From 45c20d03663878f3508eaa9b961cb0cb12cc5574 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 24 Jul 2024 21:48:38 +0200 Subject: Got post exporting working --- src/cmd/export/main.go | 75 ++++++++++++++++ src/cmd/export/posts.go | 179 ++++++++++++++++++++++++++++++++++++++ src/gmi/posts_preprocess_funcs.go | 10 ++- src/gmi/tpl.go | 2 +- src/http/http.go | 4 +- src/http/posts.go | 2 +- src/http/tpl/image.html | 1 + src/render/methods.go | 6 +- 8 files changed, 268 insertions(+), 11 deletions(-) create mode 100644 src/cmd/export/main.go create mode 100644 src/cmd/export/posts.go diff --git a/src/cmd/export/main.go b/src/cmd/export/main.go new file mode 100644 index 0000000..1d39bc7 --- /dev/null +++ b/src/cmd/export/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + + cfgpkg "dev.mediocregopher.com/mediocre-blog.git/src/cfg" + "dev.mediocregopher.com/mediocre-blog.git/src/gmi" + "dev.mediocregopher.com/mediocre-blog.git/src/http" + "dev.mediocregopher.com/mediocre-blog.git/src/post" + "dev.mediocregopher.com/mediocre-blog.git/src/render" + "dev.mediocregopher.com/mediocre-go-lib.git/mlog" +) + +func main() { + var ( + ctx = context.Background() + cfg = cfgpkg.NewBlogCfg(cfgpkg.Params{}) + dataDir cfgpkg.DataDir + httpParams http.Params + gmiParams gmi.Params + ) + + dataDir.SetupCfg(cfg) + defer dataDir.Close() + + httpParams.SetupCfg(cfg) + gmiParams.SetupCfg(cfg) + + exportDirPath := cfg.String("export-dir-path", "", "Directory to export into") + + // initialization + err := cfg.Init(ctx) + + logger := mlog.NewLogger(nil) + defer logger.Close() + + logger.Info(ctx, "process started") + defer logger.Info(ctx, "process exiting") + + if err != nil { + logger.Fatal(ctx, "initializing", err) + } + + if *exportDirPath == "" { + logger.FatalString(ctx, "--export-dir-path is required") + } + + postSQLDB, err := post.NewSQLDB(dataDir) + if err != nil { + logger.Fatal(ctx, "initializing sql db for post data", err) + } + defer postSQLDB.Close() + + var ( + urlBuilder = render.NewURLBuilder( + gmiParams.PublicURL, + httpParams.PublicURL, + gmiParams.PublicURL, + ) + postStore = post.NewStore(postSQLDB) + //postAssetStore = asset.NewStore(postSQLDB) + //postDraftStore = post.NewDraftStore(postSQLDB) + ) + + err = exportPosts( + ctx, + logger.WithNamespace("posts"), + postStore, + urlBuilder, + *exportDirPath, + ) + if err != nil { + logger.Fatal(ctx, "Failed to export post data", err) + } +} diff --git a/src/cmd/export/posts.go b/src/cmd/export/posts.go new file mode 100644 index 0000000..f54fc2c --- /dev/null +++ b/src/cmd/export/posts.go @@ -0,0 +1,179 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + txttpl "text/template" + + gmnhg "github.com/tdemin/gmnhg" + + "dev.mediocregopher.com/mediocre-blog.git/src/gmi" + "dev.mediocregopher.com/mediocre-blog.git/src/http" + "dev.mediocregopher.com/mediocre-blog.git/src/post" + "dev.mediocregopher.com/mediocre-blog.git/src/render" + "dev.mediocregopher.com/mediocre-go-lib.git/mctx" + "dev.mediocregopher.com/mediocre-go-lib.git/mlog" +) + +var postFileExtensions = map[post.Format]string{ + post.FormatGemtext: "gmi", + post.FormatMarkdown: "md", +} + +func postTargetFormat(p post.StoredPost) post.Format { + if p.Format == post.FormatMarkdown && strings.Contains(p.Body, ` %s\n\n", post.Description)) + } + writeString(body) + + return err +} + +func writePosts( + urlBuilder render.URLBuilder, posts []post.StoredPost, path string, +) error { + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("opening file: %w", err) + } + defer func() { _ = f.Close() }() + + writeString := func(str string) { + if err != nil { + return + } + _, err = f.WriteString(str) + } + + writeString("# Posts\n\n") + for _, p := range posts { + var ( + targetFormat = postTargetFormat(p) + u = urlBuilder.Post(p.ID) + ) + + if targetFormat == post.FormatMarkdown { + u = u.HTTP() + } + + writeString(fmt.Sprintf( + "=> %s %s - %s\n\n", + u.String(), + p.PublishedAt.Format("2006-01-02"), + p.Title, + )) + } + + return err +} + +func exportPosts( + ctx context.Context, + logger *mlog.Logger, + postStore post.Store, + urlBuilder render.URLBuilder, + exportDirPath string, +) error { + var ( + postsDir = filepath.Join(exportDirPath, "posts") + preprocessFuncs = map[post.Format]post.PreprocessFunctions{ + post.FormatGemtext: gmi.PostPreprocessFuncs{URLBuilder: urlBuilder}, + post.FormatMarkdown: http.NewPostPreprocessFuncs(urlBuilder), + } + ) + + if err := os.MkdirAll(postsDir, 0755); err != nil { + return fmt.Errorf("creating posts dir %q: %w", postsDir, err) + } + + logger.Info(ctx, "Getting posts") + posts, _, err := postStore.Get(0, 10000) + if err != nil { + return fmt.Errorf("getting posts: %w", err) + } + + { + postsIndexPath := filepath.Join(postsDir, "index.gmi") + ctx := mctx.Annotate(ctx, "path", postsIndexPath) + logger.Info(ctx, "Writing posts index page") + err = writePosts(urlBuilder, posts, postsIndexPath) + if err != nil { + return fmt.Errorf("writing posts index page: %w", err) + } + } + + for _, p := range posts { + var ( + targetFormat = postTargetFormat(p) + ext = postFileExtensions[targetFormat] + postFilePath = filepath.Join(postsDir, p.ID+"."+ext) + + ctx = mctx.Annotate( + ctx, + "post_id", p.ID, + "post_title", p.Title, + "post_format", p.Format, + "target_format", targetFormat, + ) + + body = new(bytes.Buffer) + ) + + logger.Info(ctx, "Rendering post body") + tpl := txttpl.New("") + + tpl, err := tpl.Parse(p.Body) + if err != nil { + return fmt.Errorf("parsing post %q body as template: %w", p.ID, err) + } + + err = tpl.Execute(body, struct { + RootURL render.URLBuilder + post.PreprocessFunctions + }{ + urlBuilder, preprocessFuncs[targetFormat], + }) + if err != nil { + return fmt.Errorf("executing post %q body as template: %w", p.ID, err) + } + + if p.Format == post.FormatMarkdown && targetFormat == post.FormatGemtext { + gemtextBodyBytes, err := gmnhg.RenderMarkdown(body.Bytes(), 0) + if err != nil { + return fmt.Errorf("converting post %q from markdown: %w", p.ID, err) + } + body = bytes.NewBuffer(gemtextBodyBytes) + } + + if err := writePostBody(p, postFilePath, body.String()); err != nil { + return fmt.Errorf("writing post %q body to file %q: %w", p.ID, postFilePath, err) + } + } + + return nil +} diff --git a/src/gmi/posts_preprocess_funcs.go b/src/gmi/posts_preprocess_funcs.go index f8fcda6..92a9494 100644 --- a/src/gmi/posts_preprocess_funcs.go +++ b/src/gmi/posts_preprocess_funcs.go @@ -6,11 +6,13 @@ import ( "dev.mediocregopher.com/mediocre-blog.git/src/render" ) -type postPreprocessFuncs struct { - urlBuilder render.URLBuilder +// NOTE If I wasn't abandoning this codebase I would give this a proper doc and +// constructor, and make URLBuilder private. +type PostPreprocessFuncs struct { + URLBuilder render.URLBuilder } -func (f postPreprocessFuncs) Image(args ...string) (string, error) { +func (f PostPreprocessFuncs) Image(args ...string) (string, error) { var ( id = args[0] descr = "Image" @@ -21,6 +23,6 @@ func (f postPreprocessFuncs) Image(args ...string) (string, error) { } return fmt.Sprintf( - "\n=> %s %s", f.urlBuilder.Asset(id), descr, + "\n=> %s %s", f.URLBuilder.Asset(id), descr, ), nil } diff --git a/src/gmi/tpl.go b/src/gmi/tpl.go index 5ea114d..d1be26c 100644 --- a/src/gmi/tpl.go +++ b/src/gmi/tpl.go @@ -35,7 +35,7 @@ var tplFS embed.FS func (a *api) tplHandler() (gemini.Handler, error) { var ( - postPreprocessFuncs = postPreprocessFuncs{a.urlBuilder} + postPreprocessFuncs = PostPreprocessFuncs{a.urlBuilder} allTpls = template.New("") ) diff --git a/src/http/http.go b/src/http/http.go index 11b4976..addc685 100644 --- a/src/http/http.go +++ b/src/http/http.go @@ -129,7 +129,7 @@ type api struct { redirectTpl *template.Template auther Auther urlBuilder render.URLBuilder - postPreprocessFuncs postPreprocessFuncs + postPreprocessFuncs post.PreprocessFunctions } // New initializes and returns a new API instance, including setting up all @@ -157,7 +157,7 @@ func New(params Params) (API, error) { ), } - a.postPreprocessFuncs = newPostPreprocessFuncs(a.urlBuilder) + a.postPreprocessFuncs = NewPostPreprocessFuncs(a.urlBuilder) a.redirectTpl = mustParseTpl(template.New(""), "redirect.html") a.srv = &http.Server{Handler: a.handler()} diff --git a/src/http/posts.go b/src/http/posts.go index 5a295a7..30c4012 100644 --- a/src/http/posts.go +++ b/src/http/posts.go @@ -23,7 +23,7 @@ type postPreprocessFuncs struct { imageTpl *template.Template } -func newPostPreprocessFuncs(urlBuilder render.URLBuilder) postPreprocessFuncs { +func NewPostPreprocessFuncs(urlBuilder render.URLBuilder) post.PreprocessFunctions { imageTpl := template.New("image.html") imageTpl = template.Must(imageTpl.Parse(mustReadTplFile("image.html"))) return postPreprocessFuncs{urlBuilder, imageTpl} diff --git a/src/http/tpl/image.html b/src/http/tpl/image.html index 7778625..2484e4c 100644 --- a/src/http/tpl/image.html +++ b/src/http/tpl/image.html @@ -3,6 +3,7 @@ {{ .Descr }} diff --git a/src/render/methods.go b/src/render/methods.go index a28f6b5..2988d8f 100644 --- a/src/render/methods.go +++ b/src/render/methods.go @@ -200,7 +200,7 @@ type preprocessPostPayload struct { // preprocessPostBody interprets the Post's Body as a text template which may // use any of the functions found in PreprocessFunctions (all must be set). It // executes the template and writes the result to the given writer. -func (m *Methods) preprocessPostBody(into io.Writer, p post.Post) error { +func (m *Methods) PreprocessPostBody(into io.Writer, p post.Post) error { tpl := txttpl.New("") tpl, err := tpl.Parse(p.Body) @@ -221,7 +221,7 @@ func (m *Methods) preprocessPostBody(into io.Writer, p post.Post) error { func (m *Methods) PostGemtextBody(p post.StoredPost) (string, error) { buf := new(bytes.Buffer) - if err := m.preprocessPostBody(buf, p.Post); err != nil { + if err := m.PreprocessPostBody(buf, p.Post); err != nil { return "", fmt.Errorf("preprocessing post body: %w", err) } @@ -243,7 +243,7 @@ func (m *Methods) PostGemtextBody(p post.StoredPost) (string, error) { func (m *Methods) PostHTMLBody(p post.StoredPost) (template.HTML, error) { bodyBuf := new(bytes.Buffer) - if err := m.preprocessPostBody(bodyBuf, p.Post); err != nil { + if err := m.PreprocessPostBody(bodyBuf, p.Post); err != nil { return "", fmt.Errorf("preprocessing post body: %w", err) } -- cgit v1.2.3