From 16f14355502e7d70cdd78c7e29b8bd029023e6f2 Mon Sep 17 00:00:00 2001 From: JP Hastings-Spital Date: Sun, 12 Nov 2023 09:33:41 +0000 Subject: [PATCH] Prep for backfeed --- tools/syndicate/config.go | 27 +++++++---- tools/syndicate/feeds.go | 17 ++++--- tools/syndicate/main.go | 59 ++++++----------------- tools/syndicate/poster/poster.go | 37 ++++++++++++++ tools/syndicate/services/mastodon/init.go | 2 +- 5 files changed, 83 insertions(+), 59 deletions(-) diff --git a/tools/syndicate/config.go b/tools/syndicate/config.go index d99ffa1b..ad8eaed5 100644 --- a/tools/syndicate/config.go +++ b/tools/syndicate/config.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" "path" "regexp" @@ -28,12 +29,13 @@ type fileConfig struct { } type config struct { - feeds []string - services map[string]shared.Service - interactionsDir string - content []string - tagMatcher *regexp.Regexp - urlToPath func(string) string + feeds []string + services map[string]shared.Service + interactionsDir string + content []string + tagMatcher *regexp.Regexp + syndicationMatchers map[string]*regexp.Regexp + urlToPath func(string) string } func parseConfig(cfgPath string) (*config, error) { @@ -44,9 +46,10 @@ func parseConfig(cfgPath string) (*config, error) { } cfg := &config{ - feeds: []string{}, - services: make(map[string]shared.Service), - interactionsDir: cfgData.InteractionsDir, + feeds: []string{}, + services: make(map[string]shared.Service), + syndicationMatchers: make(map[string]*regexp.Regexp), + interactionsDir: cfgData.InteractionsDir, urlToPath: func(url string) string { return path.Join(cfgData.PublishRoot, strings.TrimPrefix(url, cfgData.PublishURL)) }, @@ -74,6 +77,12 @@ func parseConfig(cfgPath string) (*config, error) { return nil, err } cfg.services[name] = svc + + bf, err := svc.BackfeedMatcher() + if err != nil { + return nil, fmt.Errorf("cannot perform backfeed matching for '%s' because %w", name, err) + } + cfg.syndicationMatchers[name] = bf serviceTags = append(serviceTags, name) } cfg.tagMatcher, err = shared.TagMatcher(serviceTags) diff --git a/tools/syndicate/feeds.go b/tools/syndicate/feeds.go index 6c9283d2..6e185968 100644 --- a/tools/syndicate/feeds.go +++ b/tools/syndicate/feeds.go @@ -13,7 +13,7 @@ import ( type toPostMap map[shared.SyndicationID]shared.Post type toBackfeedMap map[string]string -func parseFeed(urlToPath func(string) string, feedReader io.Reader, tagMatcher *regexp.Regexp) ([]string, toPostMap, toBackfeedMap, error) { +func parseFeed(urlToPath func(string) string, feedReader io.Reader, tagMatcher *regexp.Regexp, syndicationMatchers map[string]*regexp.Regexp) ([]string, toPostMap, toBackfeedMap, error) { fp := gofeed.NewParser() feed, err := fp.Parse(feedReader) if err != nil { @@ -22,7 +22,7 @@ func parseFeed(urlToPath func(string) string, feedReader io.Reader, tagMatcher * toPost := make(toPostMap) toBackfeed := make(toBackfeedMap) - var services []string + services := make(map[string]struct{}) for _, item := range feed.Items { if item.Extensions == nil || item.Extensions["dc"] == nil || item.Extensions["dc"]["relation"] == nil { @@ -37,24 +37,29 @@ func parseFeed(urlToPath func(string) string, feedReader io.Reader, tagMatcher * continue } toPost[sID], err = itemToPost(item, urlToPath) - services = append(services, sID.Source) + services[sID.Source] = struct{}{} if err != nil { return nil, nil, nil, err } continue } - for _, bf := range []regexp.Regexp{} { + for sName, bf := range syndicationMatchers { if bf.MatchString(ext.Value) { toBackfeed[ext.Value] = item.Link - // TODO: Login for backfed posts + services[sName] = struct{}{} break } } } } - return services, toPost, toBackfeed, nil + var servicesList []string + for sName := range services { + servicesList = append(servicesList, sName) + } + + return servicesList, toPost, toBackfeed, nil } func itemToPost(item *gofeed.Item, urlToPath func(string) string) (shared.Post, error) { diff --git a/tools/syndicate/main.go b/tools/syndicate/main.go index 250b25b0..50e7dfed 100644 --- a/tools/syndicate/main.go +++ b/tools/syndicate/main.go @@ -1,14 +1,12 @@ package main import ( - "bytes" "fmt" - "log" "os" "strings" + "github.com/by-jp/www.byjp.me/tools/syndicate/backfeeder" "github.com/by-jp/www.byjp.me/tools/syndicate/poster" - "github.com/by-jp/www.byjp.me/tools/syndicate/shared" ) func check(err error) { @@ -26,22 +24,25 @@ func main() { check(err) pstr := poster.New(cfg.services) - // bkfd := backfeeder.New(cfg.services) + bkfd := backfeeder.New(cfg.services) for _, feed := range cfg.feeds { f, err := os.Open(feed) check(err) - services, toPost, _, err := parseFeed(cfg.urlToPath, f, cfg.tagMatcher) + defer f.Close() + services, toPost, toBackfeed, err := parseFeed(cfg.urlToPath, f, cfg.tagMatcher, cfg.syndicationMatchers) check(err) - fmt.Fprintf(os.Stderr, "Found %d syndications to complete in %s\n", len(toPost), feed) - fmt.Fprintf(os.Stderr, "Connecting to %s to syndicate…\n", strings.Join(services, ", ")) - for _, sname := range services { - if err := pstr.Connect(sname); err != nil { - check(fmt.Errorf("couldn't connect to %s: %w", sname, err)) + if len(services) > 0 { + fmt.Fprintf(os.Stderr, "Connecting to %s to syndicate & backfeed…\n", strings.Join(services, ", ")) + for _, sname := range services { + if err := pstr.Connect(sname); err != nil { + check(fmt.Errorf("couldn't connect to %s: %w", sname, err)) + } } } + fmt.Fprintf(os.Stderr, "Found %d new syndications to post in %s\n", len(toPost), feed) for k, p := range toPost { if err := pstr.Post(k, p); err == nil { fmt.Printf("Posted '%s' to %s: %s\n", p.Title, k.Source, pstr.PostedURL(k)) @@ -51,40 +52,12 @@ func main() { } for _, fname := range cfg.content { - f, err := os.ReadFile(fname) - if err != nil { - fmt.Fprintf(os.Stderr, "Couldn't read %s: %v\n", fname, err) - continue - } - - tags := cfg.tagMatcher.FindAllSubmatch(f, -1) - if tags == nil { - continue - } - - var urls []string - for _, tag := range tags { - sid := shared.SyndicationID{Source: string(tag[1]), ID: string(tag[2])} - if url := pstr.PostedURL(sid); url != "" { - urls = append(urls, url) - f = bytes.ReplaceAll(f, sid.Bytes(), []byte(url)) - log.Printf("Replacing syndication tag '%s' with post URL: %s", sid, url) - } - } - - if len(urls) == 0 { - continue - } - - if err := os.WriteFile(fname, f, 0644); err != nil { - fmt.Fprintf( - os.Stderr, - "Couldn't insert posted URLs (%s) into %s: %v\n", - strings.Join(urls, ", "), - fname, - err, - ) + if err := pstr.ReplaceReferences(fname, cfg.tagMatcher); err != nil { + fmt.Fprintf(os.Stderr, "Couldn't replace syndication references: %v\n", err) } } + + fmt.Fprintf(os.Stderr, "Found %d existing syndications to backfeed from %s\n", len(toBackfeed), feed) + _ = bkfd } } diff --git a/tools/syndicate/poster/poster.go b/tools/syndicate/poster/poster.go index 6ff58871..c8c9a2b9 100644 --- a/tools/syndicate/poster/poster.go +++ b/tools/syndicate/poster/poster.go @@ -1,6 +1,12 @@ package poster import ( + "bytes" + "fmt" + "os" + "regexp" + "strings" + "github.com/by-jp/www.byjp.me/tools/syndicate/shared" ) @@ -42,3 +48,34 @@ func (p *poster) PostedURL(sid shared.SyndicationID) string { return "" } } + +func (p *poster) ReplaceReferences(fname string, tagMatcher *regexp.Regexp) error { + f, err := os.ReadFile(fname) + if err != nil { + return fmt.Errorf("couldn't read %s to replace posts within it: %w", fname, err) + } + + tags := tagMatcher.FindAllSubmatch(f, -1) + if tags == nil { + return nil + } + + var replacedURLs []string + for _, tag := range tags { + sid := shared.SyndicationID{Source: string(tag[1]), ID: string(tag[2])} + if url, ok := p.done[sid]; ok { + replacedURLs = append(replacedURLs, url) + f = bytes.ReplaceAll(f, sid.Bytes(), []byte(url)) + } + } + + if len(replacedURLs) == 0 { + return nil + } + + if err := os.WriteFile(fname, f, 0644); err != nil { + return fmt.Errorf("couldn't write %s after replacing post URLs (%s) within it: %w", fname, strings.Join(replacedURLs, ", "), err) + } + + return nil +} diff --git a/tools/syndicate/services/mastodon/init.go b/tools/syndicate/services/mastodon/init.go index 3f6f8f55..82d80af8 100644 --- a/tools/syndicate/services/mastodon/init.go +++ b/tools/syndicate/services/mastodon/init.go @@ -47,7 +47,7 @@ func (s *service) BackfeedMatcher() (*regexp.Regexp, error) { return regexp.Compile( regexp.QuoteMeta(url) + - `(?P@[a-zA-Z0-9_-]+)/(?P[0-9]+)/`, + `(?:p/)?@?(?P[a-zA-Z0-9_-]+)/(?P[0-9]+)`, ) }