From e7c8e60bcc756f79768393718bcb176d102edf65 Mon Sep 17 00:00:00 2001 From: JP Hastings-Spital Date: Wed, 3 Apr 2024 08:20:43 +0100 Subject: [PATCH] Facebook import I kinda screwed with everything else to homogenise all the functions - the FB stuff works, but I've definitely broken some of the other importers. I can work on fixing them in the future if I need them; otherwise others can look at previous commits! --- layouts/partials/friend.html | 2 +- tools/.archiveignore | 3 + tools/.gitignore | 1 + tools/archive/facebook/exportfile.go | 162 ++++++++++++++++++++++++++ tools/archive/facebook/main.go | 44 +++++++ tools/archive/facebook/types.go | 84 ++++++++++++++ tools/archive/goodreads/go.mod | 5 - tools/archive/goodreads/go.sum | 4 - tools/archive/instagram/go.mod | 8 -- tools/archive/instagram/go.sum | 4 - tools/archive/instagram/main.go | 41 ++----- tools/archive/twitter/go.mod | 14 --- tools/archive/twitter/go.sum | 43 ------- tools/{syndicate => }/go.mod | 46 ++++++-- tools/{syndicate => }/go.sum | 73 ++++++++---- tools/go.work | 9 -- tools/go.work.sum | 5 - tools/import/omnivore/go.mod | 5 - tools/import/omnivore/go.sum | 2 - tools/import/webmentionio/go.mod | 11 -- tools/import/webmentionio/go.sum | 4 - tools/import/webmentionio/main.go | 6 +- tools/shared/config.go | 67 +++++++++++ tools/shared/helpers.go | 25 ++++ tools/shared/mediamap.go | 82 +++++++++++++ tools/shared/postmapping.go | 157 +++++++++++++++++++++++++ tools/shared/references.go | 168 +++++++++++++++++++++++++++ tools/shared/types.go | 35 ++++++ 28 files changed, 927 insertions(+), 183 deletions(-) create mode 100644 tools/.archiveignore create mode 100644 tools/archive/facebook/exportfile.go create mode 100644 tools/archive/facebook/main.go create mode 100644 tools/archive/facebook/types.go delete mode 100644 tools/archive/goodreads/go.mod delete mode 100644 tools/archive/goodreads/go.sum delete mode 100644 tools/archive/instagram/go.mod delete mode 100644 tools/archive/twitter/go.mod delete mode 100644 tools/archive/twitter/go.sum rename tools/{syndicate => }/go.mod (66%) rename tools/{syndicate => }/go.sum (87%) delete mode 100644 tools/go.work delete mode 100644 tools/go.work.sum delete mode 100644 tools/import/omnivore/go.mod delete mode 100644 tools/import/omnivore/go.sum delete mode 100644 tools/import/webmentionio/go.mod delete mode 100644 tools/import/webmentionio/go.sum create mode 100644 tools/shared/config.go create mode 100644 tools/shared/helpers.go create mode 100644 tools/shared/mediamap.go create mode 100644 tools/shared/postmapping.go create mode 100644 tools/shared/references.go create mode 100644 tools/shared/types.go diff --git a/layouts/partials/friend.html b/layouts/partials/friend.html index 7ae3cf4d..80671214 100644 --- a/layouts/partials/friend.html +++ b/layouts/partials/friend.html @@ -2,5 +2,5 @@ {{- with index site.Data.friends (index $parts 0) -}} {{.name}} {{- else -}} - {{ errorf "Could not display friend %s" . }} + {{ . }} {{- end }} \ No newline at end of file diff --git a/tools/.archiveignore b/tools/.archiveignore new file mode 100644 index 00000000..8dcc3b69 --- /dev/null +++ b/tools/.archiveignore @@ -0,0 +1,3 @@ +rtv2nwevsrb7jskkoi2niza6qq +tuoyt24xgoknswodnhizqehx2u +lsdjotoggybiqsetlzmwcpptay diff --git a/tools/.gitignore b/tools/.gitignore index 4c49bd78..536cb223 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -1 +1,2 @@ .env +.reference-cache diff --git a/tools/archive/facebook/exportfile.go b/tools/archive/facebook/exportfile.go new file mode 100644 index 00000000..cc26ef60 --- /dev/null +++ b/tools/archive/facebook/exportfile.go @@ -0,0 +1,162 @@ +package main + +import ( + "crypto/md5" + "encoding/base32" + "fmt" + "os" + "path" + "regexp" + "strings" + "time" + + "github.com/bmatcuk/doublestar/v4" + "github.com/by-jp/www.byjp.me/tools/shared" + "golang.org/x/text/encoding/charmap" +) + +var replaceCountRE = regexp.MustCompile(`-\d{3}\.zip`) + +func findArchives(archiveFile string) ([]string, error) { + dir := path.Dir(archiveFile) + file := replaceCountRE.ReplaceAllString(path.Base(archiveFile), "-*.zip") + files, err := doublestar.Glob(os.DirFS(dir), file) + if err != nil { + return nil, err + } + for i, f := range files { + files[i] = path.Join(dir, f) + } + return files, nil +} + +func makePostPath(postType string, postDate time.Time, postHash string) string { + return path.Join("content", postType+"s", "facebook", postDate.Format("2006-01"), postHash) +} + +func postize(e PostCheckinPhotoOrVideo, matches []string) (shared.Post, shared.MediaMap, bool, error) { + archiveRoot := matches[1] + + postHash, err := postHash(e) + if err != nil { + return shared.Post{}, nil, false, err + } + if shared.ShouldIgnoreHash(postHash) { + return shared.Post{}, nil, true, err + } + + postDate := time.Unix(int64(e.Timestamp), 0).UTC() + post := shared.Post{ + Path: makePostPath("post", postDate, postHash), + FrontMatter: shared.FrontMatter{ + Title: e.Title, + Date: postDate.Format(time.RFC3339), + Tags: []string{ + "from-facebook", + }, + }, + } + + for _, attach := range e.Attachments { + linkURLs := attach.Data.GetSubstrings("external_context", "url") + for _, link := range linkURLs { + if link == "" { + continue + } + + if strings.HasPrefix(link, "http://") { + link = "https://" + link[7:] + } + + if strings.HasPrefix(link, "https://360.io/") || strings.HasPrefix(link, "https://fb.thisismyjam.com") { + return shared.Post{}, nil, false, err + } + + post.Path = makePostPath("bookmark", postDate, postHash) + post.FrontMatter.BookmarkOf = link + ref, err := shared.GetReferenceWithCache(link) + if err != nil { + fmt.Printf("Couldn't get reference for %s because %v\n", link, err) + } + post.FrontMatter.References = append(post.FrontMatter.References, ref) + + if strings.HasSuffix(post.FrontMatter.Title, "shared a link.") { + post.FrontMatter.Title = "" + } + } + } + + mm := make(shared.MediaMap) + var media []shared.MediaItem + for _, attach := range e.Attachments { + mediaPaths := attach.Data.GetSubstrings("media", "uri") + for _, mediaPath := range mediaPaths { + + // Ignore instagram cross-posts + if strings.Contains(mediaPath, "Instagram") { + return shared.Post{}, nil, true, nil + } + + mediaName := fmt.Sprintf("media-%d%s", len(media), path.Ext(mediaPath)) + post.FrontMatter.Media = append(post.FrontMatter.Media, shared.MediaItem{URL: mediaName}) + post.Path = makePostPath("photo", postDate, postHash) + mm[path.Join(archiveRoot, mediaPath)] = path.Join(post.Path, mediaName) + } + } + + postFile, err := formatPostText(e.Data.GetString("post"), e.Tags) + if err != nil { + return shared.Post{}, nil, false, err + } + + post.PostFile = postFile + + // Ignore my posts on others' walls + if strings.HasPrefix(post.FrontMatter.Title, "JP Hastings-Spital wrote on") { + return post, nil, true, nil + } + + return post, mm, false, nil +} + +func formatPostText(post string, tags []Tag) (string, error) { + postText, err := fixEncoding(post) + if err != nil { + return "", err + } + + names := make([]string, len(tags)) + if len(tags) > 0 { + for i, tag := range tags { + name := tag.Name + if newName, ok := tagMap[name]; ok { + name = newName + } + names[i] = fmt.Sprintf(`{{< friend "%s" >}}`, name) + } + + postText = fmt.Sprintf("%s\n\n(with %s)", postText, strings.Join(names, ", ")) + } + + return postText, nil +} + +var tagMap = map[string]string{ + "Chris Hastings-Spital": "chris", + "JP Hastings-Spital": "jp", + "Robert Shaw": "hungoverdrawn", + "Lenny Martin": "lenny", + "Helen HS": "Mum", +} + +func postHash(e PostCheckinPhotoOrVideo) (string, error) { + h := md5.New() + unique := fmt.Sprintf("%d||%s", e.Timestamp, e.Data.GetString("post")) + if _, err := h.Write([]byte(unique)); err != nil { + return "", err + } + b64 := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(h.Sum(nil)) + return strings.ToLower(b64), nil +} + +var fixEncoding = charmap.ISO8859_1.NewEncoder().String diff --git a/tools/archive/facebook/main.go b/tools/archive/facebook/main.go new file mode 100644 index 00000000..4991adb0 --- /dev/null +++ b/tools/archive/facebook/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "os" + "path" + "regexp" + + "github.com/by-jp/www.byjp.me/tools/shared" +) + +type Config struct { + Root string `default:"."` +} + +var postsFileRE = regexp.MustCompile( + `(facebook-\w+-\d{4}-\d{2}-\d{2}-\w{8})/your_facebook_activity/posts/your_posts__check_ins__photos_and_videos_\d+.json`, +) + +func main() { + var c Config + shared.Check(shared.LoadConfig("facebook", &c), "Unable to load config") + + if len(os.Args) < 2 { + shared.ExitMessage( + "Usage: %s facebook-yourname-YYYY-MM-DD-xxxxxx-YYYYMMDDTHHMMSS-001.zip", + path.Base(os.Args[0]), + ) + } + archives, err := findArchives(os.Args[1]) + shared.Check(err, "Couldn't find facebook archive zipfile") + + shared.Check( + shared.ExtractPostsFromArchives( + archives, + postsFileRE, + c.Root, + postize, + ), + "Unable to extract posts from archive files", + ) + + fmt.Println("Your Facebook export has been imported into your blog.") +} diff --git a/tools/archive/facebook/types.go b/tools/archive/facebook/types.go new file mode 100644 index 00000000..85da0597 --- /dev/null +++ b/tools/archive/facebook/types.go @@ -0,0 +1,84 @@ +package main + +import "time" + +type PostCheckinPhotoOrVideo struct { + Title string `json:"title"` + Timestamp int `json:"timestamp"` + Data DataSlice `json:"data"` + Attachments []struct { + Data DataSlice `json:"data"` + } `json:"attachments"` + Tags []Tag `json:"tags"` +} + +type Tag struct { + Name string `json:"name"` +} + +type DataSlice []map[string]interface{} + +func (ds DataSlice) GetString(key string) string { + for _, obj := range ds { + iface, ok := obj[key] + if !ok { + continue + } + + str, ok := iface.(string) + if !ok { + continue + } + + return str + } + + return "" +} + +func (ds DataSlice) GetTime(key string) time.Time { + for _, obj := range ds { + iface, ok := obj[key] + if !ok { + continue + } + + i, ok := iface.(int) + if !ok { + continue + } + + return time.Unix(int64(i), 0).UTC() + } + + return time.Unix(0, 0) +} + +func (ds DataSlice) GetSubstrings(key string, sub string) []string { + var substrs []string + for _, obj := range ds { + iface, ok := obj[key] + if !ok { + continue + } + + m, ok := iface.(map[string]interface{}) + if !ok { + continue + } + + si, ok := m[sub] + if !ok { + continue + } + + s, ok := si.(string) + if !ok { + continue + } + + substrs = append(substrs, s) + } + + return substrs +} diff --git a/tools/archive/goodreads/go.mod b/tools/archive/goodreads/go.mod deleted file mode 100644 index d6b5cc88..00000000 --- a/tools/archive/goodreads/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/by-jp/www.byjp.me/tools/archive/goodreads - -go 1.20 - -require gopkg.in/yaml.v2 v2.4.0 diff --git a/tools/archive/goodreads/go.sum b/tools/archive/goodreads/go.sum deleted file mode 100644 index dd0bc19f..00000000 --- a/tools/archive/goodreads/go.sum +++ /dev/null @@ -1,4 +0,0 @@ -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/tools/archive/instagram/go.mod b/tools/archive/instagram/go.mod deleted file mode 100644 index c5db1cbe..00000000 --- a/tools/archive/instagram/go.mod +++ /dev/null @@ -1,8 +0,0 @@ -module github.com/by-jp/www.byjp.me/tools/archive/instagram - -go 1.20 - -require ( - golang.org/x/text v0.9.0 - gopkg.in/yaml.v2 v2.4.0 -) diff --git a/tools/archive/instagram/go.sum b/tools/archive/instagram/go.sum index 74dcee37..bf420ace 100644 --- a/tools/archive/instagram/go.sum +++ b/tools/archive/instagram/go.sum @@ -1,9 +1,5 @@ -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/tools/archive/instagram/main.go b/tools/archive/instagram/main.go index c98726ee..aa07f003 100644 --- a/tools/archive/instagram/main.go +++ b/tools/archive/instagram/main.go @@ -17,6 +17,8 @@ import ( "golang.org/x/text/encoding/charmap" "gopkg.in/yaml.v2" + + . "github.com/by-jp/www.byjp.me/tools/shared" ) const titleLength = 48 @@ -48,35 +50,12 @@ var tagMap = map[string]string{ "@yvetteedrei": "Yvette", } -func check(err error, msg string) { - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n %v", msg, err) - os.Exit(1) - } -} - -type closer interface { - Close() error -} - -func doClose(c closer, msg string) { - check(c.Close(), msg) -} - type location struct { Name string Latitude float64 Longitude float64 } -type frontMatter struct { - Title string - Media []string - Date string - Draft bool - Tags []string -} - type post struct { Media []media Title string @@ -99,13 +78,13 @@ func main() { outputDir := path.Join(hugo, "content", "photos") zf, err := zip.OpenReader(archive) - check(err, "Unable to open instagram archive") - defer doClose(zf, "Unable to close zipfile") + Check(err, "Unable to open instagram archive") + defer DoClose(zf, "Unable to close zipfile") postCount, mediaMap, err := createPosts(zf, outputDir) - check(err, "Unable to create hugo posts for your instagram data") + Check(err, "Unable to create hugo posts for your instagram data") // TODO: Rewind zip? - check(copyMedia(zf, mediaMap), "Unable to copy media to your hugo blog") + Check(copyMedia(zf, mediaMap), "Unable to copy media to your hugo blog") fmt.Printf("Success! %d Instagram posts (with %d images and videos) were added to your hugo blog.\n", postCount, len(mediaMap)) } @@ -123,7 +102,7 @@ func createPosts(zf *zip.ReadCloser, outputDir string) (int, map[string]string, if err != nil { return 0, nil, err } - defer doClose(jf, "Unable to close posts file within archive") + defer DoClose(jf, "Unable to close posts file within archive") return postsFromFile(jf, outputDir) } @@ -182,7 +161,7 @@ func postToPost(p post, mediaMap map[string]string, outputDir string) error { return err } - fm := frontMatter{ + fm := FrontMatter{ Tags: []string{"imported", "from-instagram"}, } @@ -287,13 +266,13 @@ func copyMedia(zf *zip.ReadCloser, mediaMap map[string]string) error { if err != nil { return err } - defer doClose(mf, "Unable to close media file within archive") + defer DoClose(mf, "Unable to close media file within archive") mediaFile, err := os.Create(dst) if err != nil { return err } - defer doClose(mediaFile, "Unable to close media file in blog archive") + defer DoClose(mediaFile, "Unable to close media file in blog archive") io.Copy(mediaFile, mf) diff --git a/tools/archive/twitter/go.mod b/tools/archive/twitter/go.mod deleted file mode 100644 index 31a76108..00000000 --- a/tools/archive/twitter/go.mod +++ /dev/null @@ -1,14 +0,0 @@ -module github.com/by-jp/www.byjp.me/tools/archive/twitter - -go 1.20 - -require ( - github.com/PuerkitoBio/goquery v1.8.1 - golang.org/x/text v0.9.0 - gopkg.in/yaml.v2 v2.4.0 -) - -require ( - github.com/andybalholm/cascadia v1.3.1 // indirect - golang.org/x/net v0.7.0 // indirect -) diff --git a/tools/archive/twitter/go.sum b/tools/archive/twitter/go.sum deleted file mode 100644 index 331212ab..00000000 --- a/tools/archive/twitter/go.sum +++ /dev/null @@ -1,43 +0,0 @@ -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/tools/syndicate/go.mod b/tools/go.mod similarity index 66% rename from tools/syndicate/go.mod rename to tools/go.mod index 1af96df2..6c841ea6 100644 --- a/tools/syndicate/go.mod +++ b/tools/go.mod @@ -1,26 +1,35 @@ -module github.com/by-jp/www.byjp.me/tools/syndicate +module github.com/by-jp/www.byjp.me/tools -go 1.21.1 +go 1.21.6 require ( github.com/BurntSushi/toml v1.3.2 github.com/Davincible/goinsta/v3 v3.2.6 + github.com/PuerkitoBio/goquery v1.9.1 + github.com/aws/smithy-go v1.20.2 + github.com/bmatcuk/doublestar v1.3.4 github.com/bmatcuk/doublestar/v4 v4.6.1 - github.com/grokify/html-strip-tags-go v0.0.1 + github.com/grokify/html-strip-tags-go v0.1.0 github.com/h2non/filetype v1.1.3 + github.com/joho/godotenv v1.5.1 github.com/karalabe/go-bluesky v0.0.0-20230506152134-dd72fcf127a8 + github.com/kelseyhightower/envconfig v1.4.0 github.com/mattn/go-mastodon v0.0.6 - github.com/mmcdole/gofeed v1.2.1 - golang.org/x/image v0.13.0 + github.com/mmcdole/gofeed v1.3.0 + golang.org/x/image v0.15.0 + golang.org/x/net v0.21.0 + golang.org/x/text v0.14.0 + gopkg.in/yaml.v2 v2.4.0 ) require ( - github.com/PuerkitoBio/goquery v1.8.0 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/benbjohnson/clock v1.1.0 // indirect github.com/bluesky-social/indigo v0.0.0-20230504025040-8915cccc3319 // indirect github.com/chromedp/cdproto v0.0.0-20220901095120-1a01299a2163 // indirect github.com/chromedp/chromedp v0.8.5 // indirect github.com/chromedp/sysutil v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.1.0 // indirect @@ -29,12 +38,14 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v0.9.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-block-format v0.1.2 // indirect github.com/ipfs/go-cid v0.4.0 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect + github.com/ipfs/go-detect-race v0.0.1 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect @@ -46,11 +57,15 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/kr/pretty v0.2.0 // indirect + github.com/kr/text v0.1.0 // indirect + github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.18 // indirect github.com/minio/sha256-simd v1.0.0 // indirect - github.com/mmcdole/goxpp v1.1.0 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -60,18 +75,25 @@ require ( github.com/multiformats/go-multihash v0.2.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect + github.com/smartystreets/assertions v1.2.0 // indirect + github.com/smartystreets/goconvey v1.7.2 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.8.2 // indirect github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect + github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230331140348-1f892b517e70 // indirect go.uber.org/atomic v1.10.0 // indirect + go.uber.org/goleak v1.1.11 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.7.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/crypto v0.19.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/tools/syndicate/go.sum b/tools/go.sum similarity index 87% rename from tools/syndicate/go.sum rename to tools/go.sum index 9fc8b910..2712c181 100644 --- a/tools/syndicate/go.sum +++ b/tools/go.sum @@ -4,16 +4,20 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/Davincible/goinsta/v3 v3.2.6 h1:+lNIWU6NABWd2VSGe83UQypnef+kzWwjmfgGihPbwD8= github.com/Davincible/goinsta/v3 v3.2.6/go.mod h1:jIDhrWZmttL/gtXj/mkCaZyeNdAAqW3UYjasOUW0YEw= -github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= -github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= +github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bluesky-social/indigo v0.0.0-20230504025040-8915cccc3319 h1:VCNXRXpgyK3xkaQ8fzL5WzswerwLycke4B9ggLs1uOA= github.com/bluesky-social/indigo v0.0.0-20230504025040-8915cccc3319/go.mod h1:Hc09SUJXAIujaAvq7JXxi8ZQQI887grzPkHgn4JyE1Q= +github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= +github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/brianvoe/gofakeit/v6 v6.20.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8= @@ -59,8 +63,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0= -github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78= +github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4= +github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -130,6 +134,7 @@ github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -139,6 +144,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/karalabe/go-bluesky v0.0.0-20230506152134-dd72fcf127a8 h1:reWclSFW9h01YmFZ5E0aHA9SXsBgH5h/WD3XJYvwspI= github.com/karalabe/go-bluesky v0.0.0-20230506152134-dd72fcf127a8/go.mod h1:yf6AtYJO6lI1yG196MHK6ZqqSbxzVYbt2m4cTrzA8LY= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -177,10 +184,10 @@ github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1 github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s= -github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4= -github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI= -github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= +github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -236,10 +243,14 @@ github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= @@ -258,6 +269,7 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA= @@ -286,30 +298,37 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg= -golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= +golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= +golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -318,20 +337,29 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -343,6 +371,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -358,6 +388,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/go.work b/tools/go.work deleted file mode 100644 index c07de2d3..00000000 --- a/tools/go.work +++ /dev/null @@ -1,9 +0,0 @@ -go 1.21.6 - -use ( - ./archive/goodreads - ./archive/instagram - ./archive/twitter - ./import/omnivore - ./import/webmentionio -) diff --git a/tools/go.work.sum b/tools/go.work.sum deleted file mode 100644 index 8f6e1ee1..00000000 --- a/tools/go.work.sum +++ /dev/null @@ -1,5 +0,0 @@ -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= diff --git a/tools/import/omnivore/go.mod b/tools/import/omnivore/go.mod deleted file mode 100644 index b29e522b..00000000 --- a/tools/import/omnivore/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/by-jp/www.byjp.me/tools/import/omnivore - -go 1.21.6 - -require gopkg.in/yaml.v2 v2.4.0 diff --git a/tools/import/omnivore/go.sum b/tools/import/omnivore/go.sum deleted file mode 100644 index 581a0f67..00000000 --- a/tools/import/omnivore/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/tools/import/webmentionio/go.mod b/tools/import/webmentionio/go.mod deleted file mode 100644 index 5381b5cf..00000000 --- a/tools/import/webmentionio/go.mod +++ /dev/null @@ -1,11 +0,0 @@ -module github.com/by-jp/www.byjp.me/tools/import/webmentionio - -go 1.21.6 - -require ( - github.com/by-jp/www.byjp.me/tools/syndicate v0.0.0-00010101000000-000000000000 - github.com/joho/godotenv v1.5.1 - github.com/kelseyhightower/envconfig v1.4.0 -) - -replace github.com/by-jp/www.byjp.me/tools/syndicate => ../../syndicate diff --git a/tools/import/webmentionio/go.sum b/tools/import/webmentionio/go.sum deleted file mode 100644 index 3e9cb651..00000000 --- a/tools/import/webmentionio/go.sum +++ /dev/null @@ -1,4 +0,0 @@ -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= -github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= diff --git a/tools/import/webmentionio/main.go b/tools/import/webmentionio/main.go index 237cc067..ad4ed6f3 100644 --- a/tools/import/webmentionio/main.go +++ b/tools/import/webmentionio/main.go @@ -11,8 +11,7 @@ import ( "strings" "time" - "github.com/joho/godotenv" - "github.com/kelseyhightower/envconfig" + "github.com/by-jp/www.byjp.me/tools/shared" synd "github.com/by-jp/www.byjp.me/tools/syndicate/shared" ) @@ -27,9 +26,8 @@ type Config struct { } func main() { - check(godotenv.Load()) var c Config - check(envconfig.Process("webmentionio", &c)) + check(shared.LoadConfig("webmentionio", &c)) mentions, err := retrieveMentions(c) check(err) diff --git a/tools/shared/config.go b/tools/shared/config.go new file mode 100644 index 00000000..e9f55822 --- /dev/null +++ b/tools/shared/config.go @@ -0,0 +1,67 @@ +package shared + +import ( + "fmt" + "os" + "reflect" + + "github.com/joho/godotenv" + "github.com/kelseyhightower/envconfig" +) + +func LoadConfig(prefix string, config interface{}) error { + if reflect.TypeOf(config).Kind() != reflect.Ptr { + return fmt.Errorf("config must be a struct pointer") + } + configElem := reflect.ValueOf(config).Elem() + if configElem.Kind() != reflect.Struct { + return fmt.Errorf("config must be a struct pointer") + } + + if err := godotenv.Load(); err != nil { + return err + } + + if err := envconfig.Process(prefix, config); err != nil { + return err + } + + // rootField, hasRoot := configElem.Type().FieldByName("Root") + // if !hasRoot { + // return nil + // } + + // if rootField.Type.Kind() != reflect.String { + // return fmt.Errorf("config field 'Root' must be a string, not %s", rootField.Type) + // } + + // root := rootField..String() + // fmt.Printf("%s, %T\n", root, root) + + // if !isDir(path.Join(root, "content")) { + // return fmt.Errorf( + // "root must be the root of your hugo blog (%s/content/ is not a directory)", + // root, + // ) + // } + + return nil +} + +func isFile(pathStr string) bool { + st, err := os.Stat(pathStr) + if os.IsNotExist(err) { + return false + } + + return st.Mode().IsRegular() +} + +func isDir(pathStr string) bool { + st, err := os.Stat(pathStr) + if os.IsNotExist(err) { + return false + } + + return st.IsDir() +} diff --git a/tools/shared/helpers.go b/tools/shared/helpers.go new file mode 100644 index 00000000..743ecb67 --- /dev/null +++ b/tools/shared/helpers.go @@ -0,0 +1,25 @@ +package shared + +import ( + "fmt" + "os" +) + +func Check(err error, msg string) { + if err != nil { + ExitMessage(fmt.Sprintf("%s\n %v", msg, err)) + } +} + +func ExitMessage(msg string, args ...interface{}) { + fmt.Fprintf(os.Stderr, msg+"\n", args...) + os.Exit(1) +} + +type closer interface { + Close() error +} + +func DoClose(c closer, msg string) { + Check(c.Close(), msg) +} diff --git a/tools/shared/mediamap.go b/tools/shared/mediamap.go new file mode 100644 index 00000000..49328f78 --- /dev/null +++ b/tools/shared/mediamap.go @@ -0,0 +1,82 @@ +package shared + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "os" + "os/exec" + "path" + "strings" +) + +func (mm MediaMap) CopyMedia(zf *zip.ReadCloser, rootDir string) error { + for _, f := range zf.File { + dst, ok := mm[f.Name] + if !ok { + continue + } + + mf, err := f.Open() + if err != nil { + return err + } + defer DoClose(mf, "Unable to close media file within archive") + + mediaFile, err := os.Create(path.Join(rootDir, dst)) + if err != nil { + return err + } + defer DoClose(mediaFile, "Unable to close media file in blog") + + if _, err := io.Copy(mediaFile, mf); err != nil { + return fmt.Errorf("unable to copy media file: %w", err) + } + + if strings.HasSuffix(dst, ".mp4") || strings.HasSuffix(dst, ".flv") { + if err := makeThumbnail(path.Join(rootDir, dst), path.Join(rootDir, dst)+".jpg"); err != nil { + fmt.Println(err) + } + } + + delete(mm, f.Name) + } + + if len(mm) > 0 { + var missing string + for k := range mm { + missing = k + break + } + return ErrMissingMedia{Example: missing} + } + + return nil +} + +type ErrMissingMedia struct { + Example string +} + +func (e ErrMissingMedia) Error() string { + return fmt.Sprintf("some media not available in any archive, eg: %s", e.Example) +} + +func (mm MediaMap) Merge(newMediaMap MediaMap) { + for k, v := range newMediaMap { + mm[k] = v + } +} + +func makeThumbnail(video, output string) error { + buf := new(bytes.Buffer) + cmd := exec.Command("ffmpeg", "-n", "-i", video, "-vframes", "1", "-q:v", "3", output) + cmd.Stderr = buf + + if err := cmd.Run(); err != nil { + b, _ := io.ReadAll(buf) + return fmt.Errorf("couldn't create thumbnail: %s", string(b)) + } + return nil +} diff --git a/tools/shared/postmapping.go b/tools/shared/postmapping.go new file mode 100644 index 00000000..d2fcff59 --- /dev/null +++ b/tools/shared/postmapping.go @@ -0,0 +1,157 @@ +package shared + +import ( + "archive/zip" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "regexp" + "strings" + + "gopkg.in/yaml.v2" +) + +var ignoreHashes = map[string]struct{}{} + +func init() { + if ig, err := os.ReadFile(".archiveignore"); err == nil { + for _, hash := range strings.Split(string(ig), "\n") { + ignoreHashes[hash] = struct{}{} + } + } +} + +func ShouldIgnoreHash(hash string) bool { + _, ok := ignoreHashes[hash] + return ok +} + +func ExtractPostsFromArchives[ArchiveEntry interface{}]( + archives []string, + postsFileRE *regexp.Regexp, + hugoDir string, + postize func(ArchiveEntry, []string) (Post, MediaMap, bool, error), +) error { + mediaMap := make(MediaMap) + + for i, archive := range archives { + zf, err := zip.OpenReader(archive) + if err != nil { + return fmt.Errorf("unable to open archive (%s): %w", archive, err) + } + defer DoClose(zf, "Unable to close archive zipfile") + + readErr := readEntriesFromArchive(zf, postsFileRE, func(ae ArchiveEntry, matches []string) error { + post, mm, ignore, err := postize(ae, matches) + if err != nil { + return err + } + if ignore { + return nil + } + + if err := post.WriteTo(hugoDir); err != nil { + return err + } + mediaMap.Merge(mm) + + return nil + }) + if readErr != nil { + return readErr + } + + if err := mediaMap.CopyMedia(zf, hugoDir); err != nil { + _, isMissingMedia := err.(ErrMissingMedia) + if !isMissingMedia || i >= len(archives)-1 { + return err + } + } + } + return nil +} + +var ErrNoPostsFile = errors.New("no posts file found in the archive zipfile") + +func readEntriesFromArchive[ArchiveEntry interface{}]( + zf *zip.ReadCloser, + postsFileRE *regexp.Regexp, + processArchiveEntry func(ArchiveEntry, []string) error, +) error { + for _, f := range zf.File { + matches := postsFileRE.FindStringSubmatch(f.Name) + if len(matches) == 0 { + continue + } + + jf, err := f.Open() + if err != nil { + return err + } + defer DoClose(jf, "Unable to close posts file within archive") + + var entry ArchiveEntry + entries, err := readEntriesFromArchiveJSON(jf, entry) + if err != nil { + return err + } + + for _, entry := range entries { + if err := processArchiveEntry(entry, matches); err != nil { + return err + } + } + } + + return nil +} + +func readEntriesFromArchiveJSON[ArchiveEntry interface{}](r io.Reader, entry ArchiveEntry) ([]ArchiveEntry, error) { + dec := json.NewDecoder(r) + + // Opening [ + tok, err := dec.Token() + if err != nil { + return nil, err + } + if fmt.Sprintf("%s", tok) != "[" { + return nil, fmt.Errorf("posts JSON starts with '%s', instead of '['", tok) + } + + var entries []ArchiveEntry + for dec.More() { + var e ArchiveEntry + if err := dec.Decode(&e); err != nil { + return nil, fmt.Errorf("unable to decode JSON: %w", err) + } + entries = append(entries, e) + } + + return entries, nil +} + +func (p Post) WriteTo(hugoDir string) error { + postRoot := path.Join(hugoDir, p.Path) + if err := os.MkdirAll(postRoot, 0755); err != nil { + return err + } + + hugoPost, err := os.Create(path.Join(postRoot, "index.md")) + if err != nil { + return err + } + + fmt.Fprintln(hugoPost, "---") + + if err := yaml.NewEncoder(hugoPost).Encode(p.FrontMatter); err != nil { + return err + } + + fmt.Fprintln(hugoPost, "---") + fmt.Fprintln(hugoPost, p.PostFile) + + return nil +} diff --git a/tools/shared/references.go b/tools/shared/references.go new file mode 100644 index 00000000..8f2dcdc6 --- /dev/null +++ b/tools/shared/references.go @@ -0,0 +1,168 @@ +package shared + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path" + "regexp" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "gopkg.in/yaml.v2" +) + +var cacheLocation = "./.reference-cache" + +func init() { + if !isDir(cacheLocation) { + Check(os.Mkdir(cacheLocation, 0755), "couldn't create cache directory") + } +} + +var pathRe = regexp.MustCompile(`[^\w\-. ]+`) + +var maxRedirects = 5 +var withRedirects = &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= maxRedirects { + return fmt.Errorf("more than %d redirects", maxRedirects) + } + return nil + }, + Timeout: 2 * time.Second, +} + +func GetReferenceWithCache(url string) (Reference, error) { + ref := Reference{ + URL: url, + Date: time.Now(), + } + + safePath := path.Join( + cacheLocation, + pathRe.ReplaceAllString(url, "/"), + "reference.yaml", + ) + failPath := safePath + ".fail" + if err := os.MkdirAll(path.Dir(safePath), 0755); err != nil { + return ref, err + } + + if refFile, err := os.ReadFile(safePath); err == nil { + return ref, yaml.Unmarshal(refFile, &ref) + } + if failFile, err := os.ReadFile(failPath); err == nil { + return ref, fmt.Errorf(string(failFile)) + } + + if err := getRef(&ref); err != nil { + os.WriteFile(failPath, []byte(err.Error()), 0644) + return ref, err + } + + refData, err := yaml.Marshal(ref) + if err != nil { + return ref, err + } + + if err := os.WriteFile(safePath, refData, 0644); err != nil { + return ref, err + } + + return ref, nil +} + +func getRef(ref *Reference) error { + res, err := withRedirects.Get(ref.URL) + if err != nil { + return err + } + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("got HTTP %d from %s", res.StatusCode, ref.URL) + } + + contentType := res.Header.Get("Content-Type") + if !strings.HasPrefix(contentType, "text/html") { + return fmt.Errorf("is %s not HTML", contentType) + } + + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + return err + } + + oembedURL := findAttr(doc, "href", `link[rel="alternate"][type="application/json+oembed"]`) + if oembedURL != "" { + if err := getRefFromOembed(ref, oembedURL); err != nil { + fmt.Printf("OEmbed failed: %v\n", err) + } else { + return nil + } + } + + ref.Type = findAttr(doc, "content", `meta[property="og:type"]`) + ref.Author = findAttr(doc, "content", `meta[itemprop="name"]`) + ref.Name = findAttr(doc, "content", `meta[property="og:title"]`, `meta[name="twitter:title"]`) + if ref.Name == "" { + ref.Name = findAttr(doc, "", `title`) + } + + return nil +} + +func findAttr(doc *goquery.Document, attr string, selectors ...string) string { + for _, selector := range selectors { + var val string + doc.Find(selector).Each(func(i int, s *goquery.Selection) { + if s == nil || val != "" { + return + } + + if attr == "" { + val = s.Text() + } else { + if attrVal, ok := s.Attr(attr); ok { + val = attrVal + } + } + }) + + if val != "" { + return val + } + } + + return "" +} + +type oembed struct { + Title string `json:"title"` + Author string `json:"author_name"` + Type string `json:"type"` + Thumbnail string `json:"thumbnail_url"` +} + +func getRefFromOembed(ref *Reference, url string) error { + res, err := withRedirects.Get(url) + if err != nil { + return err + } + if res.StatusCode != http.StatusOK { + return fmt.Errorf("oembed http status was %d", res.StatusCode) + } + + var oe oembed + if err := json.NewDecoder(res.Body).Decode(&oe); err != nil { + return err + } + + ref.Author = oe.Author + ref.Type = oe.Type + ref.Name = oe.Title + + return nil +} diff --git a/tools/shared/types.go b/tools/shared/types.go new file mode 100644 index 00000000..e2b91f37 --- /dev/null +++ b/tools/shared/types.go @@ -0,0 +1,35 @@ +package shared + +import "time" + +// Maps where a file in an archive file is to where it should be moved in the blog +type MediaMap map[string]string + +// Represents the frontmatter of a post +type FrontMatter struct { + Title string `yaml:"title,omitempty"` + Media []MediaItem `yaml:"media,omitempty"` + Date string + Tags []string `yaml:"tags,omitempty"` + BookmarkOf string `yaml:"bookmarkOf,omitempty"` + References []Reference `yaml:"references,omitempty"` +} + +type Reference struct { + URL string `yaml:"url"` + Type string `yaml:"type"` + Name string `yaml:"name"` + Author string `yaml:"author"` + Date time.Time `yaml:"date"` +} + +type MediaItem struct { + URL string `yaml:"url"` + Alt string `yaml:"alt,omitempty"` +} + +type Post struct { + Path string + PostFile string + FrontMatter FrontMatter +}