Prep for backfeed

tools/syndicate/config.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"regexp"
@@ -28,12 +29,13 @@ type fileConfig struct {
 }
 
 type config struct {
-	feeds           []string
+	feeds               []string
-	services        map[string]shared.Service
+	services            map[string]shared.Service
-	interactionsDir string
+	interactionsDir     string
-	content         []string
+	content             []string
-	tagMatcher      *regexp.Regexp
+	tagMatcher          *regexp.Regexp
-	urlToPath       func(string) string
+	syndicationMatchers map[string]*regexp.Regexp
+	urlToPath           func(string) string
 }
 
 func parseConfig(cfgPath string) (*config, error) {
@@ -44,9 +46,10 @@ func parseConfig(cfgPath string) (*config, error) {
 	}
 
 	cfg := &config{
-		feeds:           []string{},
+		feeds:               []string{},
-		services:        make(map[string]shared.Service),
+		services:            make(map[string]shared.Service),
-		interactionsDir: cfgData.InteractionsDir,
+		syndicationMatchers: make(map[string]*regexp.Regexp),
+		interactionsDir:     cfgData.InteractionsDir,
 		urlToPath: func(url string) string {
 			return path.Join(cfgData.PublishRoot, strings.TrimPrefix(url, cfgData.PublishURL))
 		},
@@ -74,6 +77,12 @@ func parseConfig(cfgPath string) (*config, error) {
 			return nil, err
 		}
 		cfg.services[name] = svc
+
+		bf, err := svc.BackfeedMatcher()
+		if err != nil {
+			return nil, fmt.Errorf("cannot perform backfeed matching for '%s' because %w", name, err)
+		}
+		cfg.syndicationMatchers[name] = bf
 		serviceTags = append(serviceTags, name)
 	}
 	cfg.tagMatcher, err = shared.TagMatcher(serviceTags)

tools/syndicate/feeds.go

@@ -13,7 +13,7 @@ import (
 type toPostMap map[shared.SyndicationID]shared.Post
 type toBackfeedMap map[string]string
 
-func parseFeed(urlToPath func(string) string, feedReader io.Reader, tagMatcher *regexp.Regexp) ([]string, toPostMap, toBackfeedMap, error) {
+func parseFeed(urlToPath func(string) string, feedReader io.Reader, tagMatcher *regexp.Regexp, syndicationMatchers map[string]*regexp.Regexp) ([]string, toPostMap, toBackfeedMap, error) {
 	fp := gofeed.NewParser()
 	feed, err := fp.Parse(feedReader)
 	if err != nil {
@@ -22,7 +22,7 @@ func parseFeed(urlToPath func(string) string, feedReader io.Reader, tagMatcher *
 
 	toPost := make(toPostMap)
 	toBackfeed := make(toBackfeedMap)
-	var services []string
+	services := make(map[string]struct{})
 
 	for _, item := range feed.Items {
 		if item.Extensions == nil || item.Extensions["dc"] == nil || item.Extensions["dc"]["relation"] == nil {
@@ -37,24 +37,29 @@ func parseFeed(urlToPath func(string) string, feedReader io.Reader, tagMatcher *
 					continue
 				}
 				toPost[sID], err = itemToPost(item, urlToPath)
-				services = append(services, sID.Source)
+				services[sID.Source] = struct{}{}
 				if err != nil {
 					return nil, nil, nil, err
 				}
 				continue
 			}
 
-			for _, bf := range []regexp.Regexp{} {
+			for sName, bf := range syndicationMatchers {
 				if bf.MatchString(ext.Value) {
 					toBackfeed[ext.Value] = item.Link
-					// TODO: Login for backfed posts
+					services[sName] = struct{}{}
 					break
 				}
 			}
 		}
 	}
 
-	return services, toPost, toBackfeed, nil
+	var servicesList []string
+	for sName := range services {
+		servicesList = append(servicesList, sName)
+	}
+
+	return servicesList, toPost, toBackfeed, nil
 }
 
 func itemToPost(item *gofeed.Item, urlToPath func(string) string) (shared.Post, error) {

tools/syndicate/main.go

@@ -1,14 +1,12 @@
 package main
 
 import (
-	"bytes"
 	"fmt"
-	"log"
 	"os"
 	"strings"
 
+	"github.com/by-jp/www.byjp.me/tools/syndicate/backfeeder"
 	"github.com/by-jp/www.byjp.me/tools/syndicate/poster"
-	"github.com/by-jp/www.byjp.me/tools/syndicate/shared"
 )
 
 func check(err error) {
@@ -26,22 +24,25 @@ func main() {
 	check(err)
 
 	pstr := poster.New(cfg.services)
-	// bkfd := backfeeder.New(cfg.services)
+	bkfd := backfeeder.New(cfg.services)
 
 	for _, feed := range cfg.feeds {
 		f, err := os.Open(feed)
 		check(err)
-		services, toPost, _, err := parseFeed(cfg.urlToPath, f, cfg.tagMatcher)
+		defer f.Close()
+		services, toPost, toBackfeed, err := parseFeed(cfg.urlToPath, f, cfg.tagMatcher, cfg.syndicationMatchers)
 		check(err)
 
-		fmt.Fprintf(os.Stderr, "Found %d syndications to complete in %s\n", len(toPost), feed)
+		if len(services) > 0 {
-		fmt.Fprintf(os.Stderr, "Connecting to %s to syndicate…\n", strings.Join(services, ", "))
+			fmt.Fprintf(os.Stderr, "Connecting to %s to syndicate & backfeed…\n", strings.Join(services, ", "))
-		for _, sname := range services {
+			for _, sname := range services {
-			if err := pstr.Connect(sname); err != nil {
+				if err := pstr.Connect(sname); err != nil {
-				check(fmt.Errorf("couldn't connect to %s: %w", sname, err))
+					check(fmt.Errorf("couldn't connect to %s: %w", sname, err))
+				}
 			}
 		}
 
+		fmt.Fprintf(os.Stderr, "Found %d new syndications to post in %s\n", len(toPost), feed)
 		for k, p := range toPost {
 			if err := pstr.Post(k, p); err == nil {
 				fmt.Printf("Posted '%s' to %s: %s\n", p.Title, k.Source, pstr.PostedURL(k))
@@ -51,40 +52,12 @@ func main() {
 		}
 
 		for _, fname := range cfg.content {
-			f, err := os.ReadFile(fname)
+			if err := pstr.ReplaceReferences(fname, cfg.tagMatcher); err != nil {
-			if err != nil {
+				fmt.Fprintf(os.Stderr, "Couldn't replace syndication references: %v\n", err)
-				fmt.Fprintf(os.Stderr, "Couldn't read %s: %v\n", fname, err)
-				continue
-			}
-
-			tags := cfg.tagMatcher.FindAllSubmatch(f, -1)
-			if tags == nil {
-				continue
-			}
-
-			var urls []string
-			for _, tag := range tags {
-				sid := shared.SyndicationID{Source: string(tag[1]), ID: string(tag[2])}
-				if url := pstr.PostedURL(sid); url != "" {
-					urls = append(urls, url)
-					f = bytes.ReplaceAll(f, sid.Bytes(), []byte(url))
-					log.Printf("Replacing syndication tag '%s' with post URL: %s", sid, url)
-				}
-			}
-
-			if len(urls) == 0 {
-				continue
-			}
-
-			if err := os.WriteFile(fname, f, 0644); err != nil {
-				fmt.Fprintf(
-					os.Stderr,
-					"Couldn't insert posted URLs (%s) into %s: %v\n",
-					strings.Join(urls, ", "),
-					fname,
-					err,
-				)
 			}
 		}
+
+		fmt.Fprintf(os.Stderr, "Found %d existing syndications to backfeed from %s\n", len(toBackfeed), feed)
+		_ = bkfd
 	}
 }

tools/syndicate/poster/poster.go

@@ -1,6 +1,12 @@
 package poster
 
 import (
+	"bytes"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
 	"github.com/by-jp/www.byjp.me/tools/syndicate/shared"
 )
 
@@ -42,3 +48,34 @@ func (p *poster) PostedURL(sid shared.SyndicationID) string {
 		return ""
 	}
 }
+
+func (p *poster) ReplaceReferences(fname string, tagMatcher *regexp.Regexp) error {
+	f, err := os.ReadFile(fname)
+	if err != nil {
+		return fmt.Errorf("couldn't read %s to replace posts within it: %w", fname, err)
+	}
+
+	tags := tagMatcher.FindAllSubmatch(f, -1)
+	if tags == nil {
+		return nil
+	}
+
+	var replacedURLs []string
+	for _, tag := range tags {
+		sid := shared.SyndicationID{Source: string(tag[1]), ID: string(tag[2])}
+		if url, ok := p.done[sid]; ok {
+			replacedURLs = append(replacedURLs, url)
+			f = bytes.ReplaceAll(f, sid.Bytes(), []byte(url))
+		}
+	}
+
+	if len(replacedURLs) == 0 {
+		return nil
+	}
+
+	if err := os.WriteFile(fname, f, 0644); err != nil {
+		return fmt.Errorf("couldn't write %s after replacing post URLs (%s) within it: %w", fname, strings.Join(replacedURLs, ", "), err)
+	}
+
+	return nil
+}

tools/syndicate/services/mastodon/init.go

@@ -47,7 +47,7 @@ func (s *service) BackfeedMatcher() (*regexp.Regexp, error) {
 
 	return regexp.Compile(
 		regexp.QuoteMeta(url) +
-			`(?P<username>@[a-zA-Z0-9_-]+)/(?P<id>[0-9]+)/`,
+			`(?:p/)?@?(?P<username>[a-zA-Z0-9_-]+)/(?P<id>[0-9]+)`,
 	)
 }