diff --git a/layouts/partials/friend.html b/layouts/partials/friend.html
index 7ae3cf4d..80671214 100644
--- a/layouts/partials/friend.html
+++ b/layouts/partials/friend.html
@@ -2,5 +2,5 @@
{{- with index site.Data.friends (index $parts 0) -}}
{{.name}}
{{- else -}}
- {{ errorf "Could not display friend %s" . }}
+ {{ . }}
{{- end }}
\ No newline at end of file
diff --git a/tools/.archiveignore b/tools/.archiveignore
new file mode 100644
index 00000000..8dcc3b69
--- /dev/null
+++ b/tools/.archiveignore
@@ -0,0 +1,3 @@
+rtv2nwevsrb7jskkoi2niza6qq
+tuoyt24xgoknswodnhizqehx2u
+lsdjotoggybiqsetlzmwcpptay
diff --git a/tools/.gitignore b/tools/.gitignore
index 4c49bd78..536cb223 100644
--- a/tools/.gitignore
+++ b/tools/.gitignore
@@ -1 +1,2 @@
.env
+.reference-cache
diff --git a/tools/archive/facebook/exportfile.go b/tools/archive/facebook/exportfile.go
new file mode 100644
index 00000000..cc26ef60
--- /dev/null
+++ b/tools/archive/facebook/exportfile.go
@@ -0,0 +1,162 @@
+package main
+
+import (
+ "crypto/md5"
+ "encoding/base32"
+ "fmt"
+ "os"
+ "path"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/bmatcuk/doublestar/v4"
+ "github.com/by-jp/www.byjp.me/tools/shared"
+ "golang.org/x/text/encoding/charmap"
+)
+
+var replaceCountRE = regexp.MustCompile(`-\d{3}\.zip`)
+
+func findArchives(archiveFile string) ([]string, error) {
+ dir := path.Dir(archiveFile)
+ file := replaceCountRE.ReplaceAllString(path.Base(archiveFile), "-*.zip")
+ files, err := doublestar.Glob(os.DirFS(dir), file)
+ if err != nil {
+ return nil, err
+ }
+ for i, f := range files {
+ files[i] = path.Join(dir, f)
+ }
+ return files, nil
+}
+
+func makePostPath(postType string, postDate time.Time, postHash string) string {
+ return path.Join("content", postType+"s", "facebook", postDate.Format("2006-01"), postHash)
+}
+
+func postize(e PostCheckinPhotoOrVideo, matches []string) (shared.Post, shared.MediaMap, bool, error) {
+ archiveRoot := matches[1]
+
+ postHash, err := postHash(e)
+ if err != nil {
+ return shared.Post{}, nil, false, err
+ }
+ if shared.ShouldIgnoreHash(postHash) {
+ return shared.Post{}, nil, true, err
+ }
+
+ postDate := time.Unix(int64(e.Timestamp), 0).UTC()
+ post := shared.Post{
+ Path: makePostPath("post", postDate, postHash),
+ FrontMatter: shared.FrontMatter{
+ Title: e.Title,
+ Date: postDate.Format(time.RFC3339),
+ Tags: []string{
+ "from-facebook",
+ },
+ },
+ }
+
+ for _, attach := range e.Attachments {
+ linkURLs := attach.Data.GetSubstrings("external_context", "url")
+ for _, link := range linkURLs {
+ if link == "" {
+ continue
+ }
+
+ if strings.HasPrefix(link, "http://") {
+ link = "https://" + link[7:]
+ }
+
+ if strings.HasPrefix(link, "https://360.io/") || strings.HasPrefix(link, "https://fb.thisismyjam.com") {
+ return shared.Post{}, nil, false, err
+ }
+
+ post.Path = makePostPath("bookmark", postDate, postHash)
+ post.FrontMatter.BookmarkOf = link
+ ref, err := shared.GetReferenceWithCache(link)
+ if err != nil {
+ fmt.Printf("Couldn't get reference for %s because %v\n", link, err)
+ }
+ post.FrontMatter.References = append(post.FrontMatter.References, ref)
+
+ if strings.HasSuffix(post.FrontMatter.Title, "shared a link.") {
+ post.FrontMatter.Title = ""
+ }
+ }
+ }
+
+ mm := make(shared.MediaMap)
+ var media []shared.MediaItem
+ for _, attach := range e.Attachments {
+ mediaPaths := attach.Data.GetSubstrings("media", "uri")
+ for _, mediaPath := range mediaPaths {
+
+ // Ignore instagram cross-posts
+ if strings.Contains(mediaPath, "Instagram") {
+ return shared.Post{}, nil, true, nil
+ }
+
+ mediaName := fmt.Sprintf("media-%d%s", len(media), path.Ext(mediaPath))
+ post.FrontMatter.Media = append(post.FrontMatter.Media, shared.MediaItem{URL: mediaName})
+ post.Path = makePostPath("photo", postDate, postHash)
+ mm[path.Join(archiveRoot, mediaPath)] = path.Join(post.Path, mediaName)
+ }
+ }
+
+ postFile, err := formatPostText(e.Data.GetString("post"), e.Tags)
+ if err != nil {
+ return shared.Post{}, nil, false, err
+ }
+
+ post.PostFile = postFile
+
+ // Ignore my posts on others' walls
+ if strings.HasPrefix(post.FrontMatter.Title, "JP Hastings-Spital wrote on") {
+ return post, nil, true, nil
+ }
+
+ return post, mm, false, nil
+}
+
+func formatPostText(post string, tags []Tag) (string, error) {
+ postText, err := fixEncoding(post)
+ if err != nil {
+ return "", err
+ }
+
+ names := make([]string, len(tags))
+ if len(tags) > 0 {
+ for i, tag := range tags {
+ name := tag.Name
+ if newName, ok := tagMap[name]; ok {
+ name = newName
+ }
+ names[i] = fmt.Sprintf(`{{< friend "%s" >}}`, name)
+ }
+
+ postText = fmt.Sprintf("%s\n\n(with %s)", postText, strings.Join(names, ", "))
+ }
+
+ return postText, nil
+}
+
+var tagMap = map[string]string{
+ "Chris Hastings-Spital": "chris",
+ "JP Hastings-Spital": "jp",
+ "Robert Shaw": "hungoverdrawn",
+ "Lenny Martin": "lenny",
+ "Helen HS": "Mum",
+}
+
+func postHash(e PostCheckinPhotoOrVideo) (string, error) {
+ h := md5.New()
+ unique := fmt.Sprintf("%d||%s", e.Timestamp, e.Data.GetString("post"))
+ if _, err := h.Write([]byte(unique)); err != nil {
+ return "", err
+ }
+ b64 := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(h.Sum(nil))
+ return strings.ToLower(b64), nil
+}
+
+var fixEncoding = charmap.ISO8859_1.NewEncoder().String
diff --git a/tools/archive/facebook/main.go b/tools/archive/facebook/main.go
new file mode 100644
index 00000000..4991adb0
--- /dev/null
+++ b/tools/archive/facebook/main.go
@@ -0,0 +1,44 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "path"
+ "regexp"
+
+ "github.com/by-jp/www.byjp.me/tools/shared"
+)
+
+type Config struct {
+ Root string `default:"."`
+}
+
+var postsFileRE = regexp.MustCompile(
+ `(facebook-\w+-\d{4}-\d{2}-\d{2}-\w{8})/your_facebook_activity/posts/your_posts__check_ins__photos_and_videos_\d+.json`,
+)
+
+func main() {
+ var c Config
+ shared.Check(shared.LoadConfig("facebook", &c), "Unable to load config")
+
+ if len(os.Args) < 2 {
+ shared.ExitMessage(
+ "Usage: %s facebook-yourname-YYYY-MM-DD-xxxxxx-YYYYMMDDTHHMMSS-001.zip",
+ path.Base(os.Args[0]),
+ )
+ }
+ archives, err := findArchives(os.Args[1])
+ shared.Check(err, "Couldn't find facebook archive zipfile")
+
+ shared.Check(
+ shared.ExtractPostsFromArchives(
+ archives,
+ postsFileRE,
+ c.Root,
+ postize,
+ ),
+ "Unable to extract posts from archive files",
+ )
+
+ fmt.Println("Your Facebook export has been imported into your blog.")
+}
diff --git a/tools/archive/facebook/types.go b/tools/archive/facebook/types.go
new file mode 100644
index 00000000..85da0597
--- /dev/null
+++ b/tools/archive/facebook/types.go
@@ -0,0 +1,84 @@
+package main
+
+import "time"
+
+type PostCheckinPhotoOrVideo struct {
+ Title string `json:"title"`
+ Timestamp int `json:"timestamp"`
+ Data DataSlice `json:"data"`
+ Attachments []struct {
+ Data DataSlice `json:"data"`
+ } `json:"attachments"`
+ Tags []Tag `json:"tags"`
+}
+
+type Tag struct {
+ Name string `json:"name"`
+}
+
+type DataSlice []map[string]interface{}
+
+func (ds DataSlice) GetString(key string) string {
+ for _, obj := range ds {
+ iface, ok := obj[key]
+ if !ok {
+ continue
+ }
+
+ str, ok := iface.(string)
+ if !ok {
+ continue
+ }
+
+ return str
+ }
+
+ return ""
+}
+
+func (ds DataSlice) GetTime(key string) time.Time {
+ for _, obj := range ds {
+ iface, ok := obj[key]
+ if !ok {
+ continue
+ }
+
+ i, ok := iface.(int)
+ if !ok {
+ continue
+ }
+
+ return time.Unix(int64(i), 0).UTC()
+ }
+
+ return time.Unix(0, 0)
+}
+
+func (ds DataSlice) GetSubstrings(key string, sub string) []string {
+ var substrs []string
+ for _, obj := range ds {
+ iface, ok := obj[key]
+ if !ok {
+ continue
+ }
+
+ m, ok := iface.(map[string]interface{})
+ if !ok {
+ continue
+ }
+
+ si, ok := m[sub]
+ if !ok {
+ continue
+ }
+
+ s, ok := si.(string)
+ if !ok {
+ continue
+ }
+
+ substrs = append(substrs, s)
+ }
+
+ return substrs
+}
diff --git a/tools/archive/goodreads/go.mod b/tools/archive/goodreads/go.mod
deleted file mode 100644
index d6b5cc88..00000000
--- a/tools/archive/goodreads/go.mod
+++ /dev/null
@@ -1,5 +0,0 @@
-module github.com/by-jp/www.byjp.me/tools/archive/goodreads
-
-go 1.20
-
-require gopkg.in/yaml.v2 v2.4.0
diff --git a/tools/archive/goodreads/go.sum b/tools/archive/goodreads/go.sum
deleted file mode 100644
index dd0bc19f..00000000
--- a/tools/archive/goodreads/go.sum
+++ /dev/null
@@ -1,4 +0,0 @@
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/tools/archive/instagram/go.mod b/tools/archive/instagram/go.mod
deleted file mode 100644
index c5db1cbe..00000000
--- a/tools/archive/instagram/go.mod
+++ /dev/null
@@ -1,8 +0,0 @@
-module github.com/by-jp/www.byjp.me/tools/archive/instagram
-
-go 1.20
-
-require (
- golang.org/x/text v0.9.0
- gopkg.in/yaml.v2 v2.4.0
-)
diff --git a/tools/archive/instagram/go.sum b/tools/archive/instagram/go.sum
index 74dcee37..bf420ace 100644
--- a/tools/archive/instagram/go.sum
+++ b/tools/archive/instagram/go.sum
@@ -1,9 +1,5 @@
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/tools/archive/instagram/main.go b/tools/archive/instagram/main.go
index c98726ee..aa07f003 100644
--- a/tools/archive/instagram/main.go
+++ b/tools/archive/instagram/main.go
@@ -17,6 +17,8 @@ import (
"golang.org/x/text/encoding/charmap"
"gopkg.in/yaml.v2"
+
+ . "github.com/by-jp/www.byjp.me/tools/shared"
)
const titleLength = 48
@@ -48,35 +50,12 @@ var tagMap = map[string]string{
"@yvetteedrei": "Yvette",
}
-func check(err error, msg string) {
- if err != nil {
- fmt.Fprintf(os.Stderr, "%s\n %v", msg, err)
- os.Exit(1)
- }
-}
-
-type closer interface {
- Close() error
-}
-
-func doClose(c closer, msg string) {
- check(c.Close(), msg)
-}
-
type location struct {
Name string
Latitude float64
Longitude float64
}
-type frontMatter struct {
- Title string
- Media []string
- Date string
- Draft bool
- Tags []string
-}
-
type post struct {
Media []media
Title string
@@ -99,13 +78,13 @@ func main() {
outputDir := path.Join(hugo, "content", "photos")
zf, err := zip.OpenReader(archive)
- check(err, "Unable to open instagram archive")
- defer doClose(zf, "Unable to close zipfile")
+ Check(err, "Unable to open instagram archive")
+ defer DoClose(zf, "Unable to close zipfile")
postCount, mediaMap, err := createPosts(zf, outputDir)
- check(err, "Unable to create hugo posts for your instagram data")
+ Check(err, "Unable to create hugo posts for your instagram data")
// TODO: Rewind zip?
- check(copyMedia(zf, mediaMap), "Unable to copy media to your hugo blog")
+ Check(copyMedia(zf, mediaMap), "Unable to copy media to your hugo blog")
fmt.Printf("Success! %d Instagram posts (with %d images and videos) were added to your hugo blog.\n", postCount, len(mediaMap))
}
@@ -123,7 +102,7 @@ func createPosts(zf *zip.ReadCloser, outputDir string) (int, map[string]string,
if err != nil {
return 0, nil, err
}
- defer doClose(jf, "Unable to close posts file within archive")
+ defer DoClose(jf, "Unable to close posts file within archive")
return postsFromFile(jf, outputDir)
}
@@ -182,7 +161,7 @@ func postToPost(p post, mediaMap map[string]string, outputDir string) error {
return err
}
- fm := frontMatter{
+ fm := FrontMatter{
Tags: []string{"imported", "from-instagram"},
}
@@ -287,13 +266,13 @@ func copyMedia(zf *zip.ReadCloser, mediaMap map[string]string) error {
if err != nil {
return err
}
- defer doClose(mf, "Unable to close media file within archive")
+ defer DoClose(mf, "Unable to close media file within archive")
mediaFile, err := os.Create(dst)
if err != nil {
return err
}
- defer doClose(mediaFile, "Unable to close media file in blog archive")
+ defer DoClose(mediaFile, "Unable to close media file in blog archive")
io.Copy(mediaFile, mf)
diff --git a/tools/archive/twitter/go.mod b/tools/archive/twitter/go.mod
deleted file mode 100644
index 31a76108..00000000
--- a/tools/archive/twitter/go.mod
+++ /dev/null
@@ -1,14 +0,0 @@
-module github.com/by-jp/www.byjp.me/tools/archive/twitter
-
-go 1.20
-
-require (
- github.com/PuerkitoBio/goquery v1.8.1
- golang.org/x/text v0.9.0
- gopkg.in/yaml.v2 v2.4.0
-)
-
-require (
- github.com/andybalholm/cascadia v1.3.1 // indirect
- golang.org/x/net v0.7.0 // indirect
-)
diff --git a/tools/archive/twitter/go.sum b/tools/archive/twitter/go.sum
deleted file mode 100644
index 331212ab..00000000
--- a/tools/archive/twitter/go.sum
+++ /dev/null
@@ -1,43 +0,0 @@
-github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
-github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
-github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/tools/syndicate/go.mod b/tools/go.mod
similarity index 66%
rename from tools/syndicate/go.mod
rename to tools/go.mod
index 1af96df2..6c841ea6 100644
--- a/tools/syndicate/go.mod
+++ b/tools/go.mod
@@ -1,26 +1,35 @@
-module github.com/by-jp/www.byjp.me/tools/syndicate
+module github.com/by-jp/www.byjp.me/tools
-go 1.21.1
+go 1.21.6
require (
github.com/BurntSushi/toml v1.3.2
github.com/Davincible/goinsta/v3 v3.2.6
+ github.com/PuerkitoBio/goquery v1.9.1
+ github.com/aws/smithy-go v1.20.2
+ github.com/bmatcuk/doublestar v1.3.4
github.com/bmatcuk/doublestar/v4 v4.6.1
- github.com/grokify/html-strip-tags-go v0.0.1
+ github.com/grokify/html-strip-tags-go v0.1.0
github.com/h2non/filetype v1.1.3
+ github.com/joho/godotenv v1.5.1
github.com/karalabe/go-bluesky v0.0.0-20230506152134-dd72fcf127a8
+ github.com/kelseyhightower/envconfig v1.4.0
github.com/mattn/go-mastodon v0.0.6
- github.com/mmcdole/gofeed v1.2.1
- golang.org/x/image v0.13.0
+ github.com/mmcdole/gofeed v1.3.0
+ golang.org/x/image v0.15.0
+ golang.org/x/net v0.21.0
+ golang.org/x/text v0.14.0
+ gopkg.in/yaml.v2 v2.4.0
)
require (
- github.com/PuerkitoBio/goquery v1.8.0 // indirect
- github.com/andybalholm/cascadia v1.3.1 // indirect
+ github.com/andybalholm/cascadia v1.3.2 // indirect
+ github.com/benbjohnson/clock v1.1.0 // indirect
github.com/bluesky-social/indigo v0.0.0-20230504025040-8915cccc3319 // indirect
github.com/chromedp/cdproto v0.0.0-20220901095120-1a01299a2163 // indirect
github.com/chromedp/chromedp v0.8.5 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.1.0 // indirect
@@ -29,12 +38,14 @@ require (
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+ github.com/hashicorp/go-hclog v0.9.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-block-format v0.1.2 // indirect
github.com/ipfs/go-cid v0.4.0 // indirect
github.com/ipfs/go-datastore v0.6.0 // indirect
+ github.com/ipfs/go-detect-race v0.0.1 // indirect
github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
@@ -46,11 +57,15 @@ require (
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
+ github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+ github.com/kr/pretty v0.2.0 // indirect
+ github.com/kr/text v0.1.0 // indirect
+ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
- github.com/mmcdole/goxpp v1.1.0 // indirect
+ github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
@@ -60,18 +75,25 @@ require (
github.com/multiformats/go-multihash v0.2.1 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
+ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde // indirect
github.com/pkg/errors v0.9.1 // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
+ github.com/smartystreets/assertions v1.2.0 // indirect
+ github.com/smartystreets/goconvey v1.7.2 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
+ github.com/stretchr/testify v1.8.2 // indirect
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
+ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20230331140348-1f892b517e70 // indirect
go.uber.org/atomic v1.10.0 // indirect
+ go.uber.org/goleak v1.1.11 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.24.0 // indirect
- golang.org/x/crypto v0.7.0 // indirect
- golang.org/x/net v0.9.0 // indirect
- golang.org/x/sys v0.7.0 // indirect
- golang.org/x/text v0.13.0 // indirect
+ golang.org/x/crypto v0.19.0 // indirect
+ golang.org/x/sys v0.17.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
)
diff --git a/tools/syndicate/go.sum b/tools/go.sum
similarity index 87%
rename from tools/syndicate/go.sum
rename to tools/go.sum
index 9fc8b910..2712c181 100644
--- a/tools/syndicate/go.sum
+++ b/tools/go.sum
@@ -4,16 +4,20 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/Davincible/goinsta/v3 v3.2.6 h1:+lNIWU6NABWd2VSGe83UQypnef+kzWwjmfgGihPbwD8=
github.com/Davincible/goinsta/v3 v3.2.6/go.mod h1:jIDhrWZmttL/gtXj/mkCaZyeNdAAqW3UYjasOUW0YEw=
-github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
-github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI=
+github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
-github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
-github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
+github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bluesky-social/indigo v0.0.0-20230504025040-8915cccc3319 h1:VCNXRXpgyK3xkaQ8fzL5WzswerwLycke4B9ggLs1uOA=
github.com/bluesky-social/indigo v0.0.0-20230504025040-8915cccc3319/go.mod h1:Hc09SUJXAIujaAvq7JXxi8ZQQI887grzPkHgn4JyE1Q=
+github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
+github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/brianvoe/gofakeit/v6 v6.20.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8=
@@ -59,8 +63,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0=
-github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78=
+github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4=
+github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc=
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
@@ -130,6 +134,7 @@ github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0
github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -139,6 +144,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/karalabe/go-bluesky v0.0.0-20230506152134-dd72fcf127a8 h1:reWclSFW9h01YmFZ5E0aHA9SXsBgH5h/WD3XJYvwspI=
github.com/karalabe/go-bluesky v0.0.0-20230506152134-dd72fcf127a8/go.mod h1:yf6AtYJO6lI1yG196MHK6ZqqSbxzVYbt2m4cTrzA8LY=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -177,10 +184,10 @@ github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s=
-github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4=
-github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI=
-github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
+github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
+github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -236,10 +243,14 @@ github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
@@ -258,6 +269,7 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA=
@@ -286,30 +298,37 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
-golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg=
-golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
+golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
+golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -318,20 +337,29 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -343,6 +371,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -358,6 +388,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/tools/go.work b/tools/go.work
deleted file mode 100644
index c07de2d3..00000000
--- a/tools/go.work
+++ /dev/null
@@ -1,9 +0,0 @@
-go 1.21.6
-
-use (
- ./archive/goodreads
- ./archive/instagram
- ./archive/twitter
- ./import/omnivore
- ./import/webmentionio
-)
diff --git a/tools/go.work.sum b/tools/go.work.sum
deleted file mode 100644
index 8f6e1ee1..00000000
--- a/tools/go.work.sum
+++ /dev/null
@@ -1,5 +0,0 @@
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
-golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
diff --git a/tools/import/omnivore/go.mod b/tools/import/omnivore/go.mod
deleted file mode 100644
index b29e522b..00000000
--- a/tools/import/omnivore/go.mod
+++ /dev/null
@@ -1,5 +0,0 @@
-module github.com/by-jp/www.byjp.me/tools/import/omnivore
-
-go 1.21.6
-
-require gopkg.in/yaml.v2 v2.4.0
diff --git a/tools/import/omnivore/go.sum b/tools/import/omnivore/go.sum
deleted file mode 100644
index 581a0f67..00000000
--- a/tools/import/omnivore/go.sum
+++ /dev/null
@@ -1,2 +0,0 @@
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/tools/import/webmentionio/go.mod b/tools/import/webmentionio/go.mod
deleted file mode 100644
index 5381b5cf..00000000
--- a/tools/import/webmentionio/go.mod
+++ /dev/null
@@ -1,11 +0,0 @@
-module github.com/by-jp/www.byjp.me/tools/import/webmentionio
-
-go 1.21.6
-
-require (
- github.com/by-jp/www.byjp.me/tools/syndicate v0.0.0-00010101000000-000000000000
- github.com/joho/godotenv v1.5.1
- github.com/kelseyhightower/envconfig v1.4.0
-)
-
-replace github.com/by-jp/www.byjp.me/tools/syndicate => ../../syndicate
diff --git a/tools/import/webmentionio/go.sum b/tools/import/webmentionio/go.sum
deleted file mode 100644
index 3e9cb651..00000000
--- a/tools/import/webmentionio/go.sum
+++ /dev/null
@@ -1,4 +0,0 @@
-github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
-github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
-github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
-github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
diff --git a/tools/import/webmentionio/main.go b/tools/import/webmentionio/main.go
index 237cc067..ad4ed6f3 100644
--- a/tools/import/webmentionio/main.go
+++ b/tools/import/webmentionio/main.go
@@ -11,8 +11,7 @@ import (
"strings"
"time"
- "github.com/joho/godotenv"
- "github.com/kelseyhightower/envconfig"
+ "github.com/by-jp/www.byjp.me/tools/shared"
synd "github.com/by-jp/www.byjp.me/tools/syndicate/shared"
)
@@ -27,9 +26,8 @@ type Config struct {
}
func main() {
- check(godotenv.Load())
var c Config
- check(envconfig.Process("webmentionio", &c))
+ check(shared.LoadConfig("webmentionio", &c))
mentions, err := retrieveMentions(c)
check(err)
diff --git a/tools/shared/config.go b/tools/shared/config.go
new file mode 100644
index 00000000..e9f55822
--- /dev/null
+++ b/tools/shared/config.go
@@ -0,0 +1,67 @@
+package shared
+
+import (
+ "fmt"
+ "os"
+ "reflect"
+
+ "github.com/joho/godotenv"
+ "github.com/kelseyhightower/envconfig"
+)
+
+func LoadConfig(prefix string, config interface{}) error {
+ if reflect.TypeOf(config).Kind() != reflect.Ptr {
+ return fmt.Errorf("config must be a struct pointer")
+ }
+ configElem := reflect.ValueOf(config).Elem()
+ if configElem.Kind() != reflect.Struct {
+ return fmt.Errorf("config must be a struct pointer")
+ }
+
+ if err := godotenv.Load(); err != nil {
+ return err
+ }
+
+ if err := envconfig.Process(prefix, config); err != nil {
+ return err
+ }
+
+ // rootField, hasRoot := configElem.Type().FieldByName("Root")
+ // if !hasRoot {
+ // return nil
+ // }
+
+ // if rootField.Type.Kind() != reflect.String {
+ // return fmt.Errorf("config field 'Root' must be a string, not %s", rootField.Type)
+ // }
+
+ // root := rootField..String()
+ // fmt.Printf("%s, %T\n", root, root)
+
+ // if !isDir(path.Join(root, "content")) {
+ // return fmt.Errorf(
+ // "root must be the root of your hugo blog (%s/content/ is not a directory)",
+ // root,
+ // )
+ // }
+
+ return nil
+}
+
+func isFile(pathStr string) bool {
+ st, err := os.Stat(pathStr)
+ if os.IsNotExist(err) {
+ return false
+ }
+
+ return st.Mode().IsRegular()
+}
+
+func isDir(pathStr string) bool {
+ st, err := os.Stat(pathStr)
+ if os.IsNotExist(err) {
+ return false
+ }
+
+ return st.IsDir()
+}
diff --git a/tools/shared/helpers.go b/tools/shared/helpers.go
new file mode 100644
index 00000000..743ecb67
--- /dev/null
+++ b/tools/shared/helpers.go
@@ -0,0 +1,25 @@
+package shared
+
+import (
+ "fmt"
+ "os"
+)
+
+func Check(err error, msg string) {
+ if err != nil {
+ ExitMessage(fmt.Sprintf("%s\n %v", msg, err))
+ }
+}
+
+func ExitMessage(msg string, args ...interface{}) {
+ fmt.Fprintf(os.Stderr, msg+"\n", args...)
+ os.Exit(1)
+}
+
+type closer interface {
+ Close() error
+}
+
+func DoClose(c closer, msg string) {
+ Check(c.Close(), msg)
+}
diff --git a/tools/shared/mediamap.go b/tools/shared/mediamap.go
new file mode 100644
index 00000000..49328f78
--- /dev/null
+++ b/tools/shared/mediamap.go
@@ -0,0 +1,82 @@
+package shared
+
+import (
+ "archive/zip"
+ "bytes"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path"
+ "strings"
+)
+
+func (mm MediaMap) CopyMedia(zf *zip.ReadCloser, rootDir string) error {
+ for _, f := range zf.File {
+ dst, ok := mm[f.Name]
+ if !ok {
+ continue
+ }
+
+ mf, err := f.Open()
+ if err != nil {
+ return err
+ }
+ defer DoClose(mf, "Unable to close media file within archive")
+
+ mediaFile, err := os.Create(path.Join(rootDir, dst))
+ if err != nil {
+ return err
+ }
+ defer DoClose(mediaFile, "Unable to close media file in blog")
+
+ if _, err := io.Copy(mediaFile, mf); err != nil {
+ return fmt.Errorf("unable to copy media file: %w", err)
+ }
+
+ if strings.HasSuffix(dst, ".mp4") || strings.HasSuffix(dst, ".flv") {
+ if err := makeThumbnail(path.Join(rootDir, dst), path.Join(rootDir, dst)+".jpg"); err != nil {
+ fmt.Println(err)
+ }
+ }
+
+ delete(mm, f.Name)
+ }
+
+ if len(mm) > 0 {
+ var missing string
+ for k := range mm {
+ missing = k
+ break
+ }
+ return ErrMissingMedia{Example: missing}
+ }
+
+ return nil
+}
+
+type ErrMissingMedia struct {
+ Example string
+}
+
+func (e ErrMissingMedia) Error() string {
+ return fmt.Sprintf("some media not available in any archive, eg: %s", e.Example)
+}
+
+func (mm MediaMap) Merge(newMediaMap MediaMap) {
+ for k, v := range newMediaMap {
+ mm[k] = v
+ }
+}
+
+func makeThumbnail(video, output string) error {
+ buf := new(bytes.Buffer)
+ cmd := exec.Command("ffmpeg", "-n", "-i", video, "-vframes", "1", "-q:v", "3", output)
+ cmd.Stderr = buf
+
+ if err := cmd.Run(); err != nil {
+ b, _ := io.ReadAll(buf)
+ return fmt.Errorf("couldn't create thumbnail: %s", string(b))
+ }
+ return nil
+}
diff --git a/tools/shared/postmapping.go b/tools/shared/postmapping.go
new file mode 100644
index 00000000..d2fcff59
--- /dev/null
+++ b/tools/shared/postmapping.go
@@ -0,0 +1,157 @@
+package shared
+
+import (
+ "archive/zip"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "regexp"
+ "strings"
+
+ "gopkg.in/yaml.v2"
+)
+
+var ignoreHashes = map[string]struct{}{}
+
+func init() {
+ if ig, err := os.ReadFile(".archiveignore"); err == nil {
+ for _, hash := range strings.Split(string(ig), "\n") {
+ ignoreHashes[hash] = struct{}{}
+ }
+ }
+}
+
+func ShouldIgnoreHash(hash string) bool {
+ _, ok := ignoreHashes[hash]
+ return ok
+}
+
+func ExtractPostsFromArchives[ArchiveEntry interface{}](
+ archives []string,
+ postsFileRE *regexp.Regexp,
+ hugoDir string,
+ postize func(ArchiveEntry, []string) (Post, MediaMap, bool, error),
+) error {
+ mediaMap := make(MediaMap)
+
+ for i, archive := range archives {
+ zf, err := zip.OpenReader(archive)
+ if err != nil {
+ return fmt.Errorf("unable to open archive (%s): %w", archive, err)
+ }
+ defer DoClose(zf, "Unable to close archive zipfile")
+
+ readErr := readEntriesFromArchive(zf, postsFileRE, func(ae ArchiveEntry, matches []string) error {
+ post, mm, ignore, err := postize(ae, matches)
+ if err != nil {
+ return err
+ }
+ if ignore {
+ return nil
+ }
+
+ if err := post.WriteTo(hugoDir); err != nil {
+ return err
+ }
+ mediaMap.Merge(mm)
+
+ return nil
+ })
+ if readErr != nil {
+ return readErr
+ }
+
+ if err := mediaMap.CopyMedia(zf, hugoDir); err != nil {
+ _, isMissingMedia := err.(ErrMissingMedia)
+ if !isMissingMedia || i >= len(archives)-1 {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+var ErrNoPostsFile = errors.New("no posts file found in the archive zipfile")
+
+func readEntriesFromArchive[ArchiveEntry interface{}](
+ zf *zip.ReadCloser,
+ postsFileRE *regexp.Regexp,
+ processArchiveEntry func(ArchiveEntry, []string) error,
+) error {
+ for _, f := range zf.File {
+ matches := postsFileRE.FindStringSubmatch(f.Name)
+ if len(matches) == 0 {
+ continue
+ }
+
+ jf, err := f.Open()
+ if err != nil {
+ return err
+ }
+ defer DoClose(jf, "Unable to close posts file within archive")
+
+ var entry ArchiveEntry
+ entries, err := readEntriesFromArchiveJSON(jf, entry)
+ if err != nil {
+ return err
+ }
+
+ for _, entry := range entries {
+ if err := processArchiveEntry(entry, matches); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func readEntriesFromArchiveJSON[ArchiveEntry interface{}](r io.Reader, entry ArchiveEntry) ([]ArchiveEntry, error) {
+ dec := json.NewDecoder(r)
+
+ // Opening [
+ tok, err := dec.Token()
+ if err != nil {
+ return nil, err
+ }
+ if fmt.Sprintf("%s", tok) != "[" {
+ return nil, fmt.Errorf("posts JSON starts with '%s', instead of '['", tok)
+ }
+
+ var entries []ArchiveEntry
+ for dec.More() {
+ var e ArchiveEntry
+ if err := dec.Decode(&e); err != nil {
+ return nil, fmt.Errorf("unable to decode JSON: %w", err)
+ }
+ entries = append(entries, e)
+ }
+
+ return entries, nil
+}
+
+func (p Post) WriteTo(hugoDir string) error {
+ postRoot := path.Join(hugoDir, p.Path)
+ if err := os.MkdirAll(postRoot, 0755); err != nil {
+ return err
+ }
+
+ hugoPost, err := os.Create(path.Join(postRoot, "index.md"))
+ if err != nil {
+ return err
+ }
+
+ fmt.Fprintln(hugoPost, "---")
+
+ if err := yaml.NewEncoder(hugoPost).Encode(p.FrontMatter); err != nil {
+ return err
+ }
+
+ fmt.Fprintln(hugoPost, "---")
+ fmt.Fprintln(hugoPost, p.PostFile)
+
+ return nil
+}
diff --git a/tools/shared/references.go b/tools/shared/references.go
new file mode 100644
index 00000000..8f2dcdc6
--- /dev/null
+++ b/tools/shared/references.go
@@ -0,0 +1,168 @@
+package shared
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "os"
+ "path"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/PuerkitoBio/goquery"
+ "gopkg.in/yaml.v2"
+)
+
+var cacheLocation = "./.reference-cache"
+
+func init() {
+ if !isDir(cacheLocation) {
+ Check(os.Mkdir(cacheLocation, 0755), "couldn't create cache directory")
+ }
+}
+
+var pathRe = regexp.MustCompile(`[^\w\-. ]+`)
+
+var maxRedirects = 5
+var withRedirects = &http.Client{
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ if len(via) >= maxRedirects {
+ return fmt.Errorf("more than %d redirects", maxRedirects)
+ }
+ return nil
+ },
+ Timeout: 2 * time.Second,
+}
+
+func GetReferenceWithCache(url string) (Reference, error) {
+ ref := Reference{
+ URL: url,
+ Date: time.Now(),
+ }
+
+ safePath := path.Join(
+ cacheLocation,
+ pathRe.ReplaceAllString(url, "/"),
+ "reference.yaml",
+ )
+ failPath := safePath + ".fail"
+ if err := os.MkdirAll(path.Dir(safePath), 0755); err != nil {
+ return ref, err
+ }
+
+ if refFile, err := os.ReadFile(safePath); err == nil {
+ return ref, yaml.Unmarshal(refFile, &ref)
+ }
+ if failFile, err := os.ReadFile(failPath); err == nil {
+ return ref, fmt.Errorf(string(failFile))
+ }
+
+ if err := getRef(&ref); err != nil {
+ os.WriteFile(failPath, []byte(err.Error()), 0644)
+ return ref, err
+ }
+
+ refData, err := yaml.Marshal(ref)
+ if err != nil {
+ return ref, err
+ }
+
+ if err := os.WriteFile(safePath, refData, 0644); err != nil {
+ return ref, err
+ }
+
+ return ref, nil
+}
+
+func getRef(ref *Reference) error {
+ res, err := withRedirects.Get(ref.URL)
+ if err != nil {
+ return err
+ }
+
+ if res.StatusCode != http.StatusOK {
+ return fmt.Errorf("got HTTP %d from %s", res.StatusCode, ref.URL)
+ }
+
+ contentType := res.Header.Get("Content-Type")
+ if !strings.HasPrefix(contentType, "text/html") {
+ return fmt.Errorf("is %s not HTML", contentType)
+ }
+
+ doc, err := goquery.NewDocumentFromReader(res.Body)
+ if err != nil {
+ return err
+ }
+
+ oembedURL := findAttr(doc, "href", `link[rel="alternate"][type="application/json+oembed"]`)
+ if oembedURL != "" {
+ if err := getRefFromOembed(ref, oembedURL); err != nil {
+ fmt.Printf("OEmbed failed: %v\n", err)
+ } else {
+ return nil
+ }
+ }
+
+ ref.Type = findAttr(doc, "content", `meta[property="og:type"]`)
+ ref.Author = findAttr(doc, "content", `meta[itemprop="name"]`)
+ ref.Name = findAttr(doc, "content", `meta[property="og:title"]`, `meta[name="twitter:title"]`)
+ if ref.Name == "" {
+ ref.Name = findAttr(doc, "", `title`)
+ }
+
+ return nil
+}
+
+func findAttr(doc *goquery.Document, attr string, selectors ...string) string {
+ for _, selector := range selectors {
+ var val string
+ doc.Find(selector).Each(func(i int, s *goquery.Selection) {
+ if s == nil || val != "" {
+ return
+ }
+
+ if attr == "" {
+ val = s.Text()
+ } else {
+ if attrVal, ok := s.Attr(attr); ok {
+ val = attrVal
+ }
+ }
+ })
+
+ if val != "" {
+ return val
+ }
+ }
+
+ return ""
+}
+
+type oembed struct {
+ Title string `json:"title"`
+ Author string `json:"author_name"`
+ Type string `json:"type"`
+ Thumbnail string `json:"thumbnail_url"`
+}
+
+func getRefFromOembed(ref *Reference, url string) error {
+ res, err := withRedirects.Get(url)
+ if err != nil {
+ return err
+ }
+ if res.StatusCode != http.StatusOK {
+ return fmt.Errorf("oembed http status was %d", res.StatusCode)
+ }
+
+ var oe oembed
+ if err := json.NewDecoder(res.Body).Decode(&oe); err != nil {
+ return err
+ }
+
+ ref.Author = oe.Author
+ ref.Type = oe.Type
+ ref.Name = oe.Title
+
+ return nil
+}
diff --git a/tools/shared/types.go b/tools/shared/types.go
new file mode 100644
index 00000000..e2b91f37
--- /dev/null
+++ b/tools/shared/types.go
@@ -0,0 +1,35 @@
+package shared
+
+import "time"
+
+// Maps where a file in an archive file is to where it should be moved in the blog
+type MediaMap map[string]string
+
+// Represents the frontmatter of a post
+type FrontMatter struct {
+ Title string `yaml:"title,omitempty"`
+ Media []MediaItem `yaml:"media,omitempty"`
+ Date string
+ Tags []string `yaml:"tags,omitempty"`
+ BookmarkOf string `yaml:"bookmarkOf,omitempty"`
+ References []Reference `yaml:"references,omitempty"`
+}
+
+type Reference struct {
+ URL string `yaml:"url"`
+ Type string `yaml:"type"`
+ Name string `yaml:"name"`
+ Author string `yaml:"author"`
+ Date time.Time `yaml:"date"`
+}
+
+type MediaItem struct {
+ URL string `yaml:"url"`
+ Alt string `yaml:"alt,omitempty"`
+}
+
+type Post struct {
+ Path string
+ PostFile string
+ FrontMatter FrontMatter
+}