From c9991c347d20ba9e96b3281f172a86f965934978 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 26 Jul 2024 20:47:59 +0200 Subject: Implement export script --- src/cmd/export/assets.go | 129 +++++++++++++++++++++++++++++++++++++++++++++++ src/cmd/export/main.go | 15 +++++- src/cmd/export/posts.go | 15 ++++-- src/go.mod | 3 +- src/go.sum | 3 ++ 5 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 src/cmd/export/assets.go diff --git a/src/cmd/export/assets.go b/src/cmd/export/assets.go new file mode 100644 index 0000000..2e62cbd --- /dev/null +++ b/src/cmd/export/assets.go @@ -0,0 +1,129 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "dev.mediocregopher.com/mediocre-blog.git/src/post/asset" + "dev.mediocregopher.com/mediocre-go-lib.git/mctx" + "dev.mediocregopher.com/mediocre-go-lib.git/mlog" + "github.com/nlepage/go-tarfs" +) + +func writeArchiveAsset( + assetStore asset.Store, + assetsDirPath string, + id string, +) error { + buf := new(bytes.Buffer) + if err := assetStore.Get(id, buf); err != nil { + return fmt.Errorf("loading into buffer: %w", err) + } + + gzipR, err := gzip.NewReader(buf) + if err != nil { + return fmt.Errorf("decompressing as gzip: %w", err) + } + + tarFS, err := tarfs.New(gzipR) + if err != nil { + return fmt.Errorf("parsing as tar: %w", err) + } + + return fs.WalkDir(tarFS, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("walking path %q: %w", path, err) + } else if d.IsDir() { + return nil + } + + var ( + dirPath = filepath.Join(assetsDirPath, id, filepath.Dir(path)) + dstPath = filepath.Join(dirPath, d.Name()) + srcPath = path + ) + + if err := os.MkdirAll(dirPath, 0755); err != nil { + return fmt.Errorf("creating directory %q: %w", dirPath, err) + } + + dstF, err := os.Create(dstPath) + if err != nil { + return fmt.Errorf("opening dst path %q: %w", dstPath, err) + } + defer dstF.Close() + + srcF, err := tarFS.Open(srcPath) + if err != nil { + return fmt.Errorf("opening path %q within the tar: %w", srcPath, err) + } + defer srcF.Close() + + if _, err = io.Copy(dstF, srcF); err != nil { + return fmt.Errorf("copying %q into %q: %w", srcPath, dstPath, err) + } + + return nil + }) +} + +func writeAsset( + assetStore asset.Store, + assetsDirPath string, + id string, +) error { + if strings.HasSuffix(id, ".tgz") { + return writeArchiveAsset(assetStore, assetsDirPath, id) + } + + assetPath := filepath.Join(assetsDirPath, id) + + f, err := os.Create(assetPath) + if err != nil { + return fmt.Errorf("creating file %q: %w", assetPath, err) + } + defer func() { _ = f.Close() }() + + if err := assetStore.Get(id, f); err != nil { + return fmt.Errorf("writing asset to %q: %w", assetPath, err) + } + + return nil +} + +func exportAssets( + ctx context.Context, + logger *mlog.Logger, + assetStore asset.Store, + exportDirPath string, +) error { + var ( + assetsDirPath = filepath.Join(exportDirPath, "assets") + ) + + if err := os.MkdirAll(assetsDirPath, 0755); err != nil { + return fmt.Errorf("creating asset dir %q: %w", assetsDirPath, err) + } + + logger.Info(ctx, "Listing assets") + assets, err := assetStore.List() + if err != nil { + return fmt.Errorf("listing assets: %w", err) + } + + for _, id := range assets { + logger.Info(mctx.Annotate(ctx, "assetID", id), "Writing asset") + if err := writeAsset(assetStore, assetsDirPath, id); err != nil { + return fmt.Errorf("writing asset %q: %w", id, err) + } + } + + return nil +} diff --git a/src/cmd/export/main.go b/src/cmd/export/main.go index 1d39bc7..23fa643 100644 --- a/src/cmd/export/main.go +++ b/src/cmd/export/main.go @@ -7,6 +7,7 @@ import ( "dev.mediocregopher.com/mediocre-blog.git/src/gmi" "dev.mediocregopher.com/mediocre-blog.git/src/http" "dev.mediocregopher.com/mediocre-blog.git/src/post" + "dev.mediocregopher.com/mediocre-blog.git/src/post/asset" "dev.mediocregopher.com/mediocre-blog.git/src/render" "dev.mediocregopher.com/mediocre-go-lib.git/mlog" ) @@ -57,8 +58,8 @@ func main() { httpParams.PublicURL, gmiParams.PublicURL, ) - postStore = post.NewStore(postSQLDB) - //postAssetStore = asset.NewStore(postSQLDB) + postStore = post.NewStore(postSQLDB) + postAssetStore = asset.NewStore(postSQLDB) //postDraftStore = post.NewDraftStore(postSQLDB) ) @@ -72,4 +73,14 @@ func main() { if err != nil { logger.Fatal(ctx, "Failed to export post data", err) } + + err = exportAssets( + ctx, + logger.WithNamespace("assets"), + postAssetStore, + *exportDirPath, + ) + if err != nil { + logger.Fatal(ctx, "Failed to export asset data", err) + } } diff --git a/src/cmd/export/posts.go b/src/cmd/export/posts.go index f54fc2c..569ff6b 100644 --- a/src/cmd/export/posts.go +++ b/src/cmd/export/posts.go @@ -31,7 +31,7 @@ func postTargetFormat(p post.StoredPost) post.Format { return post.FormatGemtext } -func writePostBody(post post.StoredPost, path, body string) error { +func writePostBody(p post.StoredPost, path, body string) error { f, err := os.Create(path) if err != nil { return fmt.Errorf("opening file: %w", err) @@ -45,11 +45,18 @@ func writePostBody(post post.StoredPost, path, body string) error { _, err = f.WriteString(str) } - writeString(fmt.Sprintf("# %s\n\n", post.Title)) - if post.Description != "" { - writeString(fmt.Sprintf("> %s\n\n", post.Description)) + if f := postTargetFormat(p); f == post.FormatGemtext { + writeString("=> ../posts/ Back to All Posts\n\n") + } else if f == post.FormatMarkdown { + writeString(fmt.Sprintf("---\nTitle: %q\n---\n", p.Title)) + writeString("[Back to All Posts](../posts/)\n\n") + } + writeString(fmt.Sprintf("# %s\n\n", p.Title)) + if p.Description != "" { + writeString(fmt.Sprintf("> %s\n\n", p.Description)) } writeString(body) + writeString(fmt.Sprintf("\n-----\n\nPublished %s\n", p.PublishedAt.Format("2006-01-02"))) return err } diff --git a/src/go.mod b/src/go.mod index 55b55a9..e7c077f 100644 --- a/src/go.mod +++ b/src/go.mod @@ -11,7 +11,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.8 github.com/omeid/go-tarfs v0.0.0-20171018021839-bf0d15c58b89 github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.1 github.com/tdemin/gmnhg v0.4.2 github.com/tilinna/clock v1.1.0 golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 @@ -23,6 +23,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/grokify/html-strip-tags-go v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/nlepage/go-tarfs v1.2.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect diff --git a/src/go.sum b/src/go.sum index cdd6480..3a38159 100644 --- a/src/go.sum +++ b/src/go.sum @@ -143,6 +143,8 @@ github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niklasfasching/go-org v1.5.0 h1:V8IwoSPm/d61bceyWFxxnQLtlvNT+CjiYIhtZLdnMF0= github.com/niklasfasching/go-org v1.5.0/go.mod h1:sSb8ylwnAG+h8MGFDB3R1D5bxf8wA08REfhjShg3kjA= +github.com/nlepage/go-tarfs v1.2.1 h1:o37+JPA+ajllGKSPfy5+YpsNHDjZnAoyfvf5GsUa+Ks= +github.com/nlepage/go-tarfs v1.2.1/go.mod h1:rno18mpMy9aEH1IiJVftFsqPyIpwqSUiAOpJYjlV2NA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= @@ -196,6 +198,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tdemin/gmnhg v0.4.2 h1:3g6Id0lNHT7wSIYsETBYJCEY/vWBmvof9UCI1hiq4D0= github.com/tdemin/gmnhg v0.4.2/go.mod h1:InvfH68/bP+F8No6y8wFPgrxpxfjxXQFPmvDp92HTiQ= github.com/tilinna/clock v1.1.0 h1:6IQQQCo6KoBxVudv6gwtY8o4eDfhHo8ojA5dP0MfhSs= -- cgit v1.2.3