mirror of
https://github.com/by-jp/www.byjp.me.git
synced 2025-08-09 05:36:07 +01:00
Facebook import
I kinda screwed with everything else to homogenise all the functions - the FB stuff works, but I've definitely broken some of the other importers. I can work on fixing them in the future if I need them; otherwise others can look at previous commits!
This commit is contained in:
JP Hastings-Spital
parent
4e5f0b798c
commit
e7c8e60bcc
28 changed files with 927 additions and 183 deletions
|
|
@ -2,5 +2,5 @@
|
|||
{{- with index site.Data.friends (index $parts 0) -}}
|
||||
<span class="h-card{{ range collections.Last (sub (len $parts) 1) $parts }} {{.}}{{ end }}"><a href="{{.site}}" rel="{{.xfn}}" class="u-url p-name" target="_blank">{{.name}}</a></span>
|
||||
{{- else -}}
|
||||
{{ errorf "Could not display friend %s" . }}
|
||||
{{ . }}
|
||||
{{- end }}
|
||||
3
tools/.archiveignore
Normal file
3
tools/.archiveignore
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
rtv2nwevsrb7jskkoi2niza6qq
|
||||
tuoyt24xgoknswodnhizqehx2u
|
||||
lsdjotoggybiqsetlzmwcpptay
|
||||
1
tools/.gitignore
vendored
1
tools/.gitignore
vendored
|
|
@ -1 +1,2 @@
|
|||
.env
|
||||
.reference-cache
|
||||
|
|
|
|||
162
tools/archive/facebook/exportfile.go
Normal file
162
tools/archive/facebook/exportfile.go
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/base32"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/by-jp/www.byjp.me/tools/shared"
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
)
|
||||
|
||||
var replaceCountRE = regexp.MustCompile(`-\d{3}\.zip`)
|
||||
|
||||
func findArchives(archiveFile string) ([]string, error) {
|
||||
dir := path.Dir(archiveFile)
|
||||
file := replaceCountRE.ReplaceAllString(path.Base(archiveFile), "-*.zip")
|
||||
files, err := doublestar.Glob(os.DirFS(dir), file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, f := range files {
|
||||
files[i] = path.Join(dir, f)
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func makePostPath(postType string, postDate time.Time, postHash string) string {
|
||||
return path.Join("content", postType+"s", "facebook", postDate.Format("2006-01"), postHash)
|
||||
}
|
||||
|
||||
func postize(e PostCheckinPhotoOrVideo, matches []string) (shared.Post, shared.MediaMap, bool, error) {
|
||||
archiveRoot := matches[1]
|
||||
|
||||
postHash, err := postHash(e)
|
||||
if err != nil {
|
||||
return shared.Post{}, nil, false, err
|
||||
}
|
||||
if shared.ShouldIgnoreHash(postHash) {
|
||||
return shared.Post{}, nil, true, err
|
||||
}
|
||||
|
||||
postDate := time.Unix(int64(e.Timestamp), 0).UTC()
|
||||
post := shared.Post{
|
||||
Path: makePostPath("post", postDate, postHash),
|
||||
FrontMatter: shared.FrontMatter{
|
||||
Title: e.Title,
|
||||
Date: postDate.Format(time.RFC3339),
|
||||
Tags: []string{
|
||||
"from-facebook",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, attach := range e.Attachments {
|
||||
linkURLs := attach.Data.GetSubstrings("external_context", "url")
|
||||
for _, link := range linkURLs {
|
||||
if link == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.HasPrefix(link, "http://") {
|
||||
link = "https://" + link[7:]
|
||||
}
|
||||
|
||||
if strings.HasPrefix(link, "https://360.io/") || strings.HasPrefix(link, "https://fb.thisismyjam.com") {
|
||||
return shared.Post{}, nil, false, err
|
||||
}
|
||||
|
||||
post.Path = makePostPath("bookmark", postDate, postHash)
|
||||
post.FrontMatter.BookmarkOf = link
|
||||
ref, err := shared.GetReferenceWithCache(link)
|
||||
if err != nil {
|
||||
fmt.Printf("Couldn't get reference for %s because %v\n", link, err)
|
||||
}
|
||||
post.FrontMatter.References = append(post.FrontMatter.References, ref)
|
||||
|
||||
if strings.HasSuffix(post.FrontMatter.Title, "shared a link.") {
|
||||
post.FrontMatter.Title = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mm := make(shared.MediaMap)
|
||||
var media []shared.MediaItem
|
||||
for _, attach := range e.Attachments {
|
||||
mediaPaths := attach.Data.GetSubstrings("media", "uri")
|
||||
for _, mediaPath := range mediaPaths {
|
||||
|
||||
// Ignore instagram cross-posts
|
||||
if strings.Contains(mediaPath, "Instagram") {
|
||||
return shared.Post{}, nil, true, nil
|
||||
}
|
||||
|
||||
mediaName := fmt.Sprintf("media-%d%s", len(media), path.Ext(mediaPath))
|
||||
post.FrontMatter.Media = append(post.FrontMatter.Media, shared.MediaItem{URL: mediaName})
|
||||
post.Path = makePostPath("photo", postDate, postHash)
|
||||
mm[path.Join(archiveRoot, mediaPath)] = path.Join(post.Path, mediaName)
|
||||
}
|
||||
}
|
||||
|
||||
postFile, err := formatPostText(e.Data.GetString("post"), e.Tags)
|
||||
if err != nil {
|
||||
return shared.Post{}, nil, false, err
|
||||
}
|
||||
|
||||
post.PostFile = postFile
|
||||
|
||||
// Ignore my posts on others' walls
|
||||
if strings.HasPrefix(post.FrontMatter.Title, "JP Hastings-Spital wrote on") {
|
||||
return post, nil, true, nil
|
||||
}
|
||||
|
||||
return post, mm, false, nil
|
||||
}
|
||||
|
||||
func formatPostText(post string, tags []Tag) (string, error) {
|
||||
postText, err := fixEncoding(post)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
names := make([]string, len(tags))
|
||||
if len(tags) > 0 {
|
||||
for i, tag := range tags {
|
||||
name := tag.Name
|
||||
if newName, ok := tagMap[name]; ok {
|
||||
name = newName
|
||||
}
|
||||
names[i] = fmt.Sprintf(`{{< friend "%s" >}}`, name)
|
||||
}
|
||||
|
||||
postText = fmt.Sprintf("%s\n\n(with %s)", postText, strings.Join(names, ", "))
|
||||
}
|
||||
|
||||
return postText, nil
|
||||
}
|
||||
|
||||
var tagMap = map[string]string{
|
||||
"Chris Hastings-Spital": "chris",
|
||||
"JP Hastings-Spital": "jp",
|
||||
"Robert Shaw": "hungoverdrawn",
|
||||
"Lenny Martin": "lenny",
|
||||
"Helen HS": "Mum",
|
||||
}
|
||||
|
||||
func postHash(e PostCheckinPhotoOrVideo) (string, error) {
|
||||
h := md5.New()
|
||||
unique := fmt.Sprintf("%d||%s", e.Timestamp, e.Data.GetString("post"))
|
||||
if _, err := h.Write([]byte(unique)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
b64 := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(h.Sum(nil))
|
||||
return strings.ToLower(b64), nil
|
||||
}
|
||||
|
||||
var fixEncoding = charmap.ISO8859_1.NewEncoder().String
|
||||
44
tools/archive/facebook/main.go
Normal file
44
tools/archive/facebook/main.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
|
||||
"github.com/by-jp/www.byjp.me/tools/shared"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Root string `default:"."`
|
||||
}
|
||||
|
||||
var postsFileRE = regexp.MustCompile(
|
||||
`(facebook-\w+-\d{4}-\d{2}-\d{2}-\w{8})/your_facebook_activity/posts/your_posts__check_ins__photos_and_videos_\d+.json`,
|
||||
)
|
||||
|
||||
func main() {
|
||||
var c Config
|
||||
shared.Check(shared.LoadConfig("facebook", &c), "Unable to load config")
|
||||
|
||||
if len(os.Args) < 2 {
|
||||
shared.ExitMessage(
|
||||
"Usage: %s facebook-yourname-YYYY-MM-DD-xxxxxx-YYYYMMDDTHHMMSS-001.zip",
|
||||
path.Base(os.Args[0]),
|
||||
)
|
||||
}
|
||||
archives, err := findArchives(os.Args[1])
|
||||
shared.Check(err, "Couldn't find facebook archive zipfile")
|
||||
|
||||
shared.Check(
|
||||
shared.ExtractPostsFromArchives(
|
||||
archives,
|
||||
postsFileRE,
|
||||
c.Root,
|
||||
postize,
|
||||
),
|
||||
"Unable to extract posts from archive files",
|
||||
)
|
||||
|
||||
fmt.Println("Your Facebook export has been imported into your blog.")
|
||||
}
|
||||
84
tools/archive/facebook/types.go
Normal file
84
tools/archive/facebook/types.go
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
package main
|
||||
|
||||
import "time"
|
||||
|
||||
type PostCheckinPhotoOrVideo struct {
|
||||
Title string `json:"title"`
|
||||
Timestamp int `json:"timestamp"`
|
||||
Data DataSlice `json:"data"`
|
||||
Attachments []struct {
|
||||
Data DataSlice `json:"data"`
|
||||
} `json:"attachments"`
|
||||
Tags []Tag `json:"tags"`
|
||||
}
|
||||
|
||||
type Tag struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type DataSlice []map[string]interface{}
|
||||
|
||||
func (ds DataSlice) GetString(key string) string {
|
||||
for _, obj := range ds {
|
||||
iface, ok := obj[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
str, ok := iface.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
return str
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func (ds DataSlice) GetTime(key string) time.Time {
|
||||
for _, obj := range ds {
|
||||
iface, ok := obj[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
i, ok := iface.(int)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
return time.Unix(int64(i), 0).UTC()
|
||||
}
|
||||
|
||||
return time.Unix(0, 0)
|
||||
}
|
||||
|
||||
func (ds DataSlice) GetSubstrings(key string, sub string) []string {
|
||||
var substrs []string
|
||||
for _, obj := range ds {
|
||||
iface, ok := obj[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
m, ok := iface.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
si, ok := m[sub]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
s, ok := si.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
substrs = append(substrs, s)
|
||||
}
|
||||
|
||||
return substrs
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
module github.com/by-jp/www.byjp.me/tools/archive/goodreads
|
||||
|
||||
go 1.20
|
||||
|
||||
require gopkg.in/yaml.v2 v2.4.0
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
module github.com/by-jp/www.byjp.me/tools/archive/instagram
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
golang.org/x/text v0.9.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
|
@ -1,9 +1,5 @@
|
|||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ import (
|
|||
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
"gopkg.in/yaml.v2"
|
||||
|
||||
. "github.com/by-jp/www.byjp.me/tools/shared"
|
||||
)
|
||||
|
||||
const titleLength = 48
|
||||
|
|
@ -48,35 +50,12 @@ var tagMap = map[string]string{
|
|||
"@yvetteedrei": "Yvette",
|
||||
}
|
||||
|
||||
func check(err error, msg string) {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s\n %v", msg, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
type closer interface {
|
||||
Close() error
|
||||
}
|
||||
|
||||
func doClose(c closer, msg string) {
|
||||
check(c.Close(), msg)
|
||||
}
|
||||
|
||||
type location struct {
|
||||
Name string
|
||||
Latitude float64
|
||||
Longitude float64
|
||||
}
|
||||
|
||||
type frontMatter struct {
|
||||
Title string
|
||||
Media []string
|
||||
Date string
|
||||
Draft bool
|
||||
Tags []string
|
||||
}
|
||||
|
||||
type post struct {
|
||||
Media []media
|
||||
Title string
|
||||
|
|
@ -99,13 +78,13 @@ func main() {
|
|||
outputDir := path.Join(hugo, "content", "photos")
|
||||
|
||||
zf, err := zip.OpenReader(archive)
|
||||
check(err, "Unable to open instagram archive")
|
||||
defer doClose(zf, "Unable to close zipfile")
|
||||
Check(err, "Unable to open instagram archive")
|
||||
defer DoClose(zf, "Unable to close zipfile")
|
||||
|
||||
postCount, mediaMap, err := createPosts(zf, outputDir)
|
||||
check(err, "Unable to create hugo posts for your instagram data")
|
||||
Check(err, "Unable to create hugo posts for your instagram data")
|
||||
// TODO: Rewind zip?
|
||||
check(copyMedia(zf, mediaMap), "Unable to copy media to your hugo blog")
|
||||
Check(copyMedia(zf, mediaMap), "Unable to copy media to your hugo blog")
|
||||
|
||||
fmt.Printf("Success! %d Instagram posts (with %d images and videos) were added to your hugo blog.\n", postCount, len(mediaMap))
|
||||
}
|
||||
|
|
@ -123,7 +102,7 @@ func createPosts(zf *zip.ReadCloser, outputDir string) (int, map[string]string,
|
|||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
defer doClose(jf, "Unable to close posts file within archive")
|
||||
defer DoClose(jf, "Unable to close posts file within archive")
|
||||
|
||||
return postsFromFile(jf, outputDir)
|
||||
}
|
||||
|
|
@ -182,7 +161,7 @@ func postToPost(p post, mediaMap map[string]string, outputDir string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
fm := frontMatter{
|
||||
fm := FrontMatter{
|
||||
Tags: []string{"imported", "from-instagram"},
|
||||
}
|
||||
|
||||
|
|
@ -287,13 +266,13 @@ func copyMedia(zf *zip.ReadCloser, mediaMap map[string]string) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer doClose(mf, "Unable to close media file within archive")
|
||||
defer DoClose(mf, "Unable to close media file within archive")
|
||||
|
||||
mediaFile, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer doClose(mediaFile, "Unable to close media file in blog archive")
|
||||
defer DoClose(mediaFile, "Unable to close media file in blog archive")
|
||||
|
||||
io.Copy(mediaFile, mf)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
module github.com/by-jp/www.byjp.me/tools/archive/twitter
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.1
|
||||
golang.org/x/text v0.9.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
golang.org/x/net v0.7.0 // indirect
|
||||
)
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
|
||||
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
|
|
@ -1,26 +1,35 @@
|
|||
module github.com/by-jp/www.byjp.me/tools/syndicate
|
||||
module github.com/by-jp/www.byjp.me/tools
|
||||
|
||||
go 1.21.1
|
||||
go 1.21.6
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.3.2
|
||||
github.com/Davincible/goinsta/v3 v3.2.6
|
||||
github.com/PuerkitoBio/goquery v1.9.1
|
||||
github.com/aws/smithy-go v1.20.2
|
||||
github.com/bmatcuk/doublestar v1.3.4
|
||||
github.com/bmatcuk/doublestar/v4 v4.6.1
|
||||
github.com/grokify/html-strip-tags-go v0.0.1
|
||||
github.com/grokify/html-strip-tags-go v0.1.0
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/karalabe/go-bluesky v0.0.0-20230506152134-dd72fcf127a8
|
||||
github.com/kelseyhightower/envconfig v1.4.0
|
||||
github.com/mattn/go-mastodon v0.0.6
|
||||
github.com/mmcdole/gofeed v1.2.1
|
||||
golang.org/x/image v0.13.0
|
||||
github.com/mmcdole/gofeed v1.3.0
|
||||
golang.org/x/image v0.15.0
|
||||
golang.org/x/net v0.21.0
|
||||
golang.org/x/text v0.14.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/bluesky-social/indigo v0.0.0-20230504025040-8915cccc3319 // indirect
|
||||
github.com/chromedp/cdproto v0.0.0-20220901095120-1a01299a2163 // indirect
|
||||
github.com/chromedp/chromedp v0.8.5 // indirect
|
||||
github.com/chromedp/sysutil v1.0.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.1.0 // indirect
|
||||
|
|
@ -29,12 +38,14 @@ require (
|
|||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v0.9.2 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/ipfs/bbloom v0.0.4 // indirect
|
||||
github.com/ipfs/go-block-format v0.1.2 // indirect
|
||||
github.com/ipfs/go-cid v0.4.0 // indirect
|
||||
github.com/ipfs/go-datastore v0.6.0 // indirect
|
||||
github.com/ipfs/go-detect-race v0.0.1 // indirect
|
||||
github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
|
||||
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
|
||||
|
|
@ -46,11 +57,15 @@ require (
|
|||
github.com/jbenet/goprocess v0.1.4 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/jtolds/gls v4.20.0+incompatible // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/kr/pretty v0.2.0 // indirect
|
||||
github.com/kr/text v0.1.0 // indirect
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-isatty v0.0.18 // indirect
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.0 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
|
|
@ -60,18 +75,25 @@ require (
|
|||
github.com/multiformats/go-multihash v0.2.1 // indirect
|
||||
github.com/multiformats/go-varint v0.0.7 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
|
||||
github.com/smartystreets/assertions v1.2.0 // indirect
|
||||
github.com/smartystreets/goconvey v1.7.2 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/stretchr/testify v1.8.2 // indirect
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
|
||||
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 // indirect
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20230331140348-1f892b517e70 // indirect
|
||||
go.uber.org/atomic v1.10.0 // indirect
|
||||
go.uber.org/goleak v1.1.11 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.24.0 // indirect
|
||||
golang.org/x/crypto v0.7.0 // indirect
|
||||
golang.org/x/net v0.9.0 // indirect
|
||||
golang.org/x/sys v0.7.0 // indirect
|
||||
golang.org/x/text v0.13.0 // indirect
|
||||
golang.org/x/crypto v0.19.0 // indirect
|
||||
golang.org/x/sys v0.17.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/blake3 v1.1.7 // indirect
|
||||
)
|
||||
|
|
@ -4,16 +4,20 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8
|
|||
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/Davincible/goinsta/v3 v3.2.6 h1:+lNIWU6NABWd2VSGe83UQypnef+kzWwjmfgGihPbwD8=
|
||||
github.com/Davincible/goinsta/v3 v3.2.6/go.mod h1:jIDhrWZmttL/gtXj/mkCaZyeNdAAqW3UYjasOUW0YEw=
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI=
|
||||
github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY=
|
||||
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
|
||||
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bluesky-social/indigo v0.0.0-20230504025040-8915cccc3319 h1:VCNXRXpgyK3xkaQ8fzL5WzswerwLycke4B9ggLs1uOA=
|
||||
github.com/bluesky-social/indigo v0.0.0-20230504025040-8915cccc3319/go.mod h1:Hc09SUJXAIujaAvq7JXxi8ZQQI887grzPkHgn4JyE1Q=
|
||||
github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
|
||||
github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
|
||||
github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
|
||||
github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/brianvoe/gofakeit/v6 v6.20.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8=
|
||||
|
|
@ -59,8 +63,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa
|
|||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
|
||||
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0=
|
||||
github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78=
|
||||
github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4=
|
||||
github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc=
|
||||
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
|
||||
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
|
||||
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
|
||||
|
|
@ -130,6 +134,7 @@ github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0
|
|||
github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
|
|
@ -139,6 +144,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
|
|||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/karalabe/go-bluesky v0.0.0-20230506152134-dd72fcf127a8 h1:reWclSFW9h01YmFZ5E0aHA9SXsBgH5h/WD3XJYvwspI=
|
||||
github.com/karalabe/go-bluesky v0.0.0-20230506152134-dd72fcf127a8/go.mod h1:yf6AtYJO6lI1yG196MHK6ZqqSbxzVYbt2m4cTrzA8LY=
|
||||
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
|
||||
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
|
|
@ -177,10 +184,10 @@ github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1
|
|||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s=
|
||||
github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4=
|
||||
github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI=
|
||||
github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
|
||||
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
|
||||
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
|
|
@ -236,10 +243,14 @@ github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3
|
|||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
|
||||
|
|
@ -258,6 +269,7 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr
|
|||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
|
||||
go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA=
|
||||
|
|
@ -286,30 +298,37 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
|
|||
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg=
|
||||
golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
|
||||
golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
|
||||
golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -318,20 +337,29 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
|
|
@ -343,6 +371,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
|
|||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
@ -358,6 +388,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
|
|||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
go 1.21.6
|
||||
|
||||
use (
|
||||
./archive/goodreads
|
||||
./archive/instagram
|
||||
./archive/twitter
|
||||
./import/omnivore
|
||||
./import/webmentionio
|
||||
)
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
module github.com/by-jp/www.byjp.me/tools/import/omnivore
|
||||
|
||||
go 1.21.6
|
||||
|
||||
require gopkg.in/yaml.v2 v2.4.0
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
module github.com/by-jp/www.byjp.me/tools/import/webmentionio
|
||||
|
||||
go 1.21.6
|
||||
|
||||
require (
|
||||
github.com/by-jp/www.byjp.me/tools/syndicate v0.0.0-00010101000000-000000000000
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/kelseyhightower/envconfig v1.4.0
|
||||
)
|
||||
|
||||
replace github.com/by-jp/www.byjp.me/tools/syndicate => ../../syndicate
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
|
||||
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
|
||||
|
|
@ -11,8 +11,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/kelseyhightower/envconfig"
|
||||
"github.com/by-jp/www.byjp.me/tools/shared"
|
||||
|
||||
synd "github.com/by-jp/www.byjp.me/tools/syndicate/shared"
|
||||
)
|
||||
|
|
@ -27,9 +26,8 @@ type Config struct {
|
|||
}
|
||||
|
||||
func main() {
|
||||
check(godotenv.Load())
|
||||
var c Config
|
||||
check(envconfig.Process("webmentionio", &c))
|
||||
check(shared.LoadConfig("webmentionio", &c))
|
||||
|
||||
mentions, err := retrieveMentions(c)
|
||||
check(err)
|
||||
|
|
|
|||
67
tools/shared/config.go
Normal file
67
tools/shared/config.go
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
package shared
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/kelseyhightower/envconfig"
|
||||
)
|
||||
|
||||
func LoadConfig(prefix string, config interface{}) error {
|
||||
if reflect.TypeOf(config).Kind() != reflect.Ptr {
|
||||
return fmt.Errorf("config must be a struct pointer")
|
||||
}
|
||||
configElem := reflect.ValueOf(config).Elem()
|
||||
if configElem.Kind() != reflect.Struct {
|
||||
return fmt.Errorf("config must be a struct pointer")
|
||||
}
|
||||
|
||||
if err := godotenv.Load(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := envconfig.Process(prefix, config); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// rootField, hasRoot := configElem.Type().FieldByName("Root")
|
||||
// if !hasRoot {
|
||||
// return nil
|
||||
// }
|
||||
|
||||
// if rootField.Type.Kind() != reflect.String {
|
||||
// return fmt.Errorf("config field 'Root' must be a string, not %s", rootField.Type)
|
||||
// }
|
||||
|
||||
// root := rootField..String()
|
||||
// fmt.Printf("%s, %T\n", root, root)
|
||||
|
||||
// if !isDir(path.Join(root, "content")) {
|
||||
// return fmt.Errorf(
|
||||
// "root must be the root of your hugo blog (%s/content/ is not a directory)",
|
||||
// root,
|
||||
// )
|
||||
// }
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isFile(pathStr string) bool {
|
||||
st, err := os.Stat(pathStr)
|
||||
if os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
|
||||
return st.Mode().IsRegular()
|
||||
}
|
||||
|
||||
func isDir(pathStr string) bool {
|
||||
st, err := os.Stat(pathStr)
|
||||
if os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
|
||||
return st.IsDir()
|
||||
}
|
||||
25
tools/shared/helpers.go
Normal file
25
tools/shared/helpers.go
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
package shared
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func Check(err error, msg string) {
|
||||
if err != nil {
|
||||
ExitMessage(fmt.Sprintf("%s\n %v", msg, err))
|
||||
}
|
||||
}
|
||||
|
||||
func ExitMessage(msg string, args ...interface{}) {
|
||||
fmt.Fprintf(os.Stderr, msg+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
type closer interface {
|
||||
Close() error
|
||||
}
|
||||
|
||||
func DoClose(c closer, msg string) {
|
||||
Check(c.Close(), msg)
|
||||
}
|
||||
82
tools/shared/mediamap.go
Normal file
82
tools/shared/mediamap.go
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
package shared
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (mm MediaMap) CopyMedia(zf *zip.ReadCloser, rootDir string) error {
|
||||
for _, f := range zf.File {
|
||||
dst, ok := mm[f.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
mf, err := f.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer DoClose(mf, "Unable to close media file within archive")
|
||||
|
||||
mediaFile, err := os.Create(path.Join(rootDir, dst))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer DoClose(mediaFile, "Unable to close media file in blog")
|
||||
|
||||
if _, err := io.Copy(mediaFile, mf); err != nil {
|
||||
return fmt.Errorf("unable to copy media file: %w", err)
|
||||
}
|
||||
|
||||
if strings.HasSuffix(dst, ".mp4") || strings.HasSuffix(dst, ".flv") {
|
||||
if err := makeThumbnail(path.Join(rootDir, dst), path.Join(rootDir, dst)+".jpg"); err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
delete(mm, f.Name)
|
||||
}
|
||||
|
||||
if len(mm) > 0 {
|
||||
var missing string
|
||||
for k := range mm {
|
||||
missing = k
|
||||
break
|
||||
}
|
||||
return ErrMissingMedia{Example: missing}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type ErrMissingMedia struct {
|
||||
Example string
|
||||
}
|
||||
|
||||
func (e ErrMissingMedia) Error() string {
|
||||
return fmt.Sprintf("some media not available in any archive, eg: %s", e.Example)
|
||||
}
|
||||
|
||||
func (mm MediaMap) Merge(newMediaMap MediaMap) {
|
||||
for k, v := range newMediaMap {
|
||||
mm[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
func makeThumbnail(video, output string) error {
|
||||
buf := new(bytes.Buffer)
|
||||
cmd := exec.Command("ffmpeg", "-n", "-i", video, "-vframes", "1", "-q:v", "3", output)
|
||||
cmd.Stderr = buf
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
b, _ := io.ReadAll(buf)
|
||||
return fmt.Errorf("couldn't create thumbnail: %s", string(b))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
157
tools/shared/postmapping.go
Normal file
157
tools/shared/postmapping.go
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
package shared
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var ignoreHashes = map[string]struct{}{}
|
||||
|
||||
func init() {
|
||||
if ig, err := os.ReadFile(".archiveignore"); err == nil {
|
||||
for _, hash := range strings.Split(string(ig), "\n") {
|
||||
ignoreHashes[hash] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ShouldIgnoreHash(hash string) bool {
|
||||
_, ok := ignoreHashes[hash]
|
||||
return ok
|
||||
}
|
||||
|
||||
func ExtractPostsFromArchives[ArchiveEntry interface{}](
|
||||
archives []string,
|
||||
postsFileRE *regexp.Regexp,
|
||||
hugoDir string,
|
||||
postize func(ArchiveEntry, []string) (Post, MediaMap, bool, error),
|
||||
) error {
|
||||
mediaMap := make(MediaMap)
|
||||
|
||||
for i, archive := range archives {
|
||||
zf, err := zip.OpenReader(archive)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open archive (%s): %w", archive, err)
|
||||
}
|
||||
defer DoClose(zf, "Unable to close archive zipfile")
|
||||
|
||||
readErr := readEntriesFromArchive(zf, postsFileRE, func(ae ArchiveEntry, matches []string) error {
|
||||
post, mm, ignore, err := postize(ae, matches)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ignore {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := post.WriteTo(hugoDir); err != nil {
|
||||
return err
|
||||
}
|
||||
mediaMap.Merge(mm)
|
||||
|
||||
return nil
|
||||
})
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
|
||||
if err := mediaMap.CopyMedia(zf, hugoDir); err != nil {
|
||||
_, isMissingMedia := err.(ErrMissingMedia)
|
||||
if !isMissingMedia || i >= len(archives)-1 {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var ErrNoPostsFile = errors.New("no posts file found in the archive zipfile")
|
||||
|
||||
func readEntriesFromArchive[ArchiveEntry interface{}](
|
||||
zf *zip.ReadCloser,
|
||||
postsFileRE *regexp.Regexp,
|
||||
processArchiveEntry func(ArchiveEntry, []string) error,
|
||||
) error {
|
||||
for _, f := range zf.File {
|
||||
matches := postsFileRE.FindStringSubmatch(f.Name)
|
||||
if len(matches) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
jf, err := f.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer DoClose(jf, "Unable to close posts file within archive")
|
||||
|
||||
var entry ArchiveEntry
|
||||
entries, err := readEntriesFromArchiveJSON(jf, entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if err := processArchiveEntry(entry, matches); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func readEntriesFromArchiveJSON[ArchiveEntry interface{}](r io.Reader, entry ArchiveEntry) ([]ArchiveEntry, error) {
|
||||
dec := json.NewDecoder(r)
|
||||
|
||||
// Opening [
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fmt.Sprintf("%s", tok) != "[" {
|
||||
return nil, fmt.Errorf("posts JSON starts with '%s', instead of '['", tok)
|
||||
}
|
||||
|
||||
var entries []ArchiveEntry
|
||||
for dec.More() {
|
||||
var e ArchiveEntry
|
||||
if err := dec.Decode(&e); err != nil {
|
||||
return nil, fmt.Errorf("unable to decode JSON: %w", err)
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (p Post) WriteTo(hugoDir string) error {
|
||||
postRoot := path.Join(hugoDir, p.Path)
|
||||
if err := os.MkdirAll(postRoot, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hugoPost, err := os.Create(path.Join(postRoot, "index.md"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Fprintln(hugoPost, "---")
|
||||
|
||||
if err := yaml.NewEncoder(hugoPost).Encode(p.FrontMatter); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Fprintln(hugoPost, "---")
|
||||
fmt.Fprintln(hugoPost, p.PostFile)
|
||||
|
||||
return nil
|
||||
}
|
||||
168
tools/shared/references.go
Normal file
168
tools/shared/references.go
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
package shared
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var cacheLocation = "./.reference-cache"
|
||||
|
||||
func init() {
|
||||
if !isDir(cacheLocation) {
|
||||
Check(os.Mkdir(cacheLocation, 0755), "couldn't create cache directory")
|
||||
}
|
||||
}
|
||||
|
||||
var pathRe = regexp.MustCompile(`[^\w\-. ]+`)
|
||||
|
||||
var maxRedirects = 5
|
||||
var withRedirects = &http.Client{
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return fmt.Errorf("more than %d redirects", maxRedirects)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Timeout: 2 * time.Second,
|
||||
}
|
||||
|
||||
func GetReferenceWithCache(url string) (Reference, error) {
|
||||
ref := Reference{
|
||||
URL: url,
|
||||
Date: time.Now(),
|
||||
}
|
||||
|
||||
safePath := path.Join(
|
||||
cacheLocation,
|
||||
pathRe.ReplaceAllString(url, "/"),
|
||||
"reference.yaml",
|
||||
)
|
||||
failPath := safePath + ".fail"
|
||||
if err := os.MkdirAll(path.Dir(safePath), 0755); err != nil {
|
||||
return ref, err
|
||||
}
|
||||
|
||||
if refFile, err := os.ReadFile(safePath); err == nil {
|
||||
return ref, yaml.Unmarshal(refFile, &ref)
|
||||
}
|
||||
if failFile, err := os.ReadFile(failPath); err == nil {
|
||||
return ref, fmt.Errorf(string(failFile))
|
||||
}
|
||||
|
||||
if err := getRef(&ref); err != nil {
|
||||
os.WriteFile(failPath, []byte(err.Error()), 0644)
|
||||
return ref, err
|
||||
}
|
||||
|
||||
refData, err := yaml.Marshal(ref)
|
||||
if err != nil {
|
||||
return ref, err
|
||||
}
|
||||
|
||||
if err := os.WriteFile(safePath, refData, 0644); err != nil {
|
||||
return ref, err
|
||||
}
|
||||
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
func getRef(ref *Reference) error {
|
||||
res, err := withRedirects.Get(ref.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("got HTTP %d from %s", res.StatusCode, ref.URL)
|
||||
}
|
||||
|
||||
contentType := res.Header.Get("Content-Type")
|
||||
if !strings.HasPrefix(contentType, "text/html") {
|
||||
return fmt.Errorf("is %s not HTML", contentType)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
oembedURL := findAttr(doc, "href", `link[rel="alternate"][type="application/json+oembed"]`)
|
||||
if oembedURL != "" {
|
||||
if err := getRefFromOembed(ref, oembedURL); err != nil {
|
||||
fmt.Printf("OEmbed failed: %v\n", err)
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ref.Type = findAttr(doc, "content", `meta[property="og:type"]`)
|
||||
ref.Author = findAttr(doc, "content", `meta[itemprop="name"]`)
|
||||
ref.Name = findAttr(doc, "content", `meta[property="og:title"]`, `meta[name="twitter:title"]`)
|
||||
if ref.Name == "" {
|
||||
ref.Name = findAttr(doc, "", `title`)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func findAttr(doc *goquery.Document, attr string, selectors ...string) string {
|
||||
for _, selector := range selectors {
|
||||
var val string
|
||||
doc.Find(selector).Each(func(i int, s *goquery.Selection) {
|
||||
if s == nil || val != "" {
|
||||
return
|
||||
}
|
||||
|
||||
if attr == "" {
|
||||
val = s.Text()
|
||||
} else {
|
||||
if attrVal, ok := s.Attr(attr); ok {
|
||||
val = attrVal
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if val != "" {
|
||||
return val
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
type oembed struct {
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author_name"`
|
||||
Type string `json:"type"`
|
||||
Thumbnail string `json:"thumbnail_url"`
|
||||
}
|
||||
|
||||
func getRefFromOembed(ref *Reference, url string) error {
|
||||
res, err := withRedirects.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("oembed http status was %d", res.StatusCode)
|
||||
}
|
||||
|
||||
var oe oembed
|
||||
if err := json.NewDecoder(res.Body).Decode(&oe); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ref.Author = oe.Author
|
||||
ref.Type = oe.Type
|
||||
ref.Name = oe.Title
|
||||
|
||||
return nil
|
||||
}
|
||||
35
tools/shared/types.go
Normal file
35
tools/shared/types.go
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
package shared
|
||||
|
||||
import "time"
|
||||
|
||||
// Maps where a file in an archive file is to where it should be moved in the blog
|
||||
type MediaMap map[string]string
|
||||
|
||||
// Represents the frontmatter of a post
|
||||
type FrontMatter struct {
|
||||
Title string `yaml:"title,omitempty"`
|
||||
Media []MediaItem `yaml:"media,omitempty"`
|
||||
Date string
|
||||
Tags []string `yaml:"tags,omitempty"`
|
||||
BookmarkOf string `yaml:"bookmarkOf,omitempty"`
|
||||
References []Reference `yaml:"references,omitempty"`
|
||||
}
|
||||
|
||||
type Reference struct {
|
||||
URL string `yaml:"url"`
|
||||
Type string `yaml:"type"`
|
||||
Name string `yaml:"name"`
|
||||
Author string `yaml:"author"`
|
||||
Date time.Time `yaml:"date"`
|
||||
}
|
||||
|
||||
type MediaItem struct {
|
||||
URL string `yaml:"url"`
|
||||
Alt string `yaml:"alt,omitempty"`
|
||||
}
|
||||
|
||||
type Post struct {
|
||||
Path string
|
||||
PostFile string
|
||||
FrontMatter FrontMatter
|
||||
}
|
||||
Loading…
Reference in a new issue