diff options
Diffstat (limited to 'srv/src')
27 files changed, 3756 insertions, 0 deletions
diff --git a/srv/src/api/api.go b/srv/src/api/api.go new file mode 100644 index 0000000..56f33b2 --- /dev/null +++ b/srv/src/api/api.go @@ -0,0 +1,188 @@ +// Package api implements the HTTP-based api for the mediocre-blog. +package api + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + + "github.com/mediocregopher/blog.mediocregopher.com/srv/cfg" + "github.com/mediocregopher/blog.mediocregopher.com/srv/chat" + "github.com/mediocregopher/blog.mediocregopher.com/srv/mailinglist" + "github.com/mediocregopher/blog.mediocregopher.com/srv/pow" + "github.com/mediocregopher/mediocre-go-lib/v2/mctx" + "github.com/mediocregopher/mediocre-go-lib/v2/mlog" +) + +// Params are used to instantiate a new API instance. All fields are required +// unless otherwise noted. +type Params struct { + Logger *mlog.Logger + PowManager pow.Manager + MailingList mailinglist.MailingList + GlobalRoom chat.Room + UserIDCalculator *chat.UserIDCalculator + + // ListenProto and ListenAddr are passed into net.Listen to create the + // API's listener. Both "tcp" and "unix" protocols are explicitly + // supported. + ListenProto, ListenAddr string + + // StaticDir and StaticProxy are mutually exclusive. + // + // If StaticDir is set then that directory on the filesystem will be used to + // serve the static site. + // + // Otherwise if StaticProxy is set all requests for the static site will be + // reverse-proxied there. + StaticDir string + StaticProxy *url.URL +} + +// SetupCfg implement the cfg.Cfger interface. +func (p *Params) SetupCfg(cfg *cfg.Cfg) { + + cfg.StringVar(&p.ListenProto, "listen-proto", "tcp", "Protocol to listen for HTTP requests with") + cfg.StringVar(&p.ListenAddr, "listen-addr", ":4000", "Address/path to listen for HTTP requests on") + + cfg.StringVar(&p.StaticDir, "static-dir", "", "Directory from which static files are served (mutually exclusive with -static-proxy-url)") + staticProxyURLStr := cfg.String("static-proxy-url", "", "HTTP address from which static files are served (mutually exclusive with -static-dir)") + + cfg.OnInit(func(ctx context.Context) error { + if *staticProxyURLStr != "" { + var err error + if p.StaticProxy, err = url.Parse(*staticProxyURLStr); err != nil { + return fmt.Errorf("parsing -static-proxy-url: %w", err) + } + + } else if p.StaticDir == "" { + return errors.New("-static-dir or -static-proxy-url is required") + } + + return nil + }) +} + +// Annotate implements mctx.Annotator interface. +func (p *Params) Annotate(a mctx.Annotations) { + a["listenProto"] = p.ListenProto + a["listenAddr"] = p.ListenAddr + + if p.StaticProxy != nil { + a["staticProxy"] = p.StaticProxy.String() + return + } + + a["staticDir"] = p.StaticDir +} + +// API will listen on the port configured for it, and serve HTTP requests for +// the mediocre-blog. +type API interface { + Shutdown(ctx context.Context) error +} + +type api struct { + params Params + srv *http.Server +} + +// New initializes and returns a new API instance, including setting up all +// listening ports. +func New(params Params) (API, error) { + + l, err := net.Listen(params.ListenProto, params.ListenAddr) + if err != nil { + return nil, fmt.Errorf("creating listen socket: %w", err) + } + + if params.ListenProto == "unix" { + if err := os.Chmod(params.ListenAddr, 0777); err != nil { + return nil, fmt.Errorf("chmod-ing unix socket: %w", err) + } + } + + a := &api{ + params: params, + } + + a.srv = &http.Server{Handler: a.handler()} + + go func() { + + err := a.srv.Serve(l) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + ctx := mctx.Annotate(context.Background(), a.params) + params.Logger.Fatal(ctx, "serving http server", err) + } + }() + + return a, nil +} + +func (a *api) Shutdown(ctx context.Context) error { + if err := a.srv.Shutdown(ctx); err != nil { + return err + } + + return nil +} + +func (a *api) handler() http.Handler { + + var staticHandler http.Handler + if a.params.StaticDir != "" { + staticHandler = http.FileServer(http.Dir(a.params.StaticDir)) + } else { + staticHandler = httputil.NewSingleHostReverseProxy(a.params.StaticProxy) + } + + staticHandler = setCSRFMiddleware(staticHandler) + + // sugar + requirePow := func(h http.Handler) http.Handler { + return a.requirePowMiddleware(h) + } + + mux := http.NewServeMux() + + mux.Handle("/", staticHandler) + + apiMux := http.NewServeMux() + apiMux.Handle("/pow/challenge", a.newPowChallengeHandler()) + apiMux.Handle("/pow/check", + requirePow( + http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {}), + ), + ) + + apiMux.Handle("/mailinglist/subscribe", requirePow(a.mailingListSubscribeHandler())) + apiMux.Handle("/mailinglist/finalize", a.mailingListFinalizeHandler()) + apiMux.Handle("/mailinglist/unsubscribe", a.mailingListUnsubscribeHandler()) + + apiMux.Handle("/chat/global/", http.StripPrefix("/chat/global", newChatHandler( + a.params.GlobalRoom, + a.params.UserIDCalculator, + a.requirePowMiddleware, + ))) + + var apiHandler http.Handler = apiMux + apiHandler = postOnlyMiddleware(apiHandler) + apiHandler = checkCSRFMiddleware(apiHandler) + apiHandler = logMiddleware(a.params.Logger, apiHandler) + apiHandler = annotateMiddleware(apiHandler) + apiHandler = addResponseHeaders(map[string]string{ + "Cache-Control": "no-store, max-age=0", + "Pragma": "no-cache", + "Expires": "0", + }, apiHandler) + + mux.Handle("/api/", http.StripPrefix("/api", apiHandler)) + + return mux +} diff --git a/srv/src/api/apiutils/apiutils.go b/srv/src/api/apiutils/apiutils.go new file mode 100644 index 0000000..223c2b9 --- /dev/null +++ b/srv/src/api/apiutils/apiutils.go @@ -0,0 +1,112 @@ +// Package apiutils contains utilities which are useful for implementing api +// endpoints. +package apiutils + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + + "github.com/mediocregopher/mediocre-go-lib/v2/mlog" +) + +type loggerCtxKey int + +// SetRequestLogger sets the given Logger onto the given Request's Context, +// returning a copy. +func SetRequestLogger(r *http.Request, logger *mlog.Logger) *http.Request { + ctx := r.Context() + ctx = context.WithValue(ctx, loggerCtxKey(0), logger) + return r.WithContext(ctx) +} + +// GetRequestLogger returns the Logger which was set by SetRequestLogger onto +// this Request, or nil. +func GetRequestLogger(r *http.Request) *mlog.Logger { + ctx := r.Context() + logger, _ := ctx.Value(loggerCtxKey(0)).(*mlog.Logger) + if logger == nil { + logger = mlog.Null + } + return logger +} + +// JSONResult writes the JSON encoding of the given value as the response body. +func JSONResult(rw http.ResponseWriter, r *http.Request, v interface{}) { + b, err := json.Marshal(v) + if err != nil { + InternalServerError(rw, r, err) + return + } + b = append(b, '\n') + + rw.Header().Set("Content-Type", "application/json") + rw.Write(b) +} + +// BadRequest writes a 400 status and a JSON encoded error struct containing the +// given error as the response body. +func BadRequest(rw http.ResponseWriter, r *http.Request, err error) { + GetRequestLogger(r).Warn(r.Context(), "bad request", err) + + rw.WriteHeader(400) + JSONResult(rw, r, struct { + Error string `json:"error"` + }{ + Error: err.Error(), + }) +} + +// InternalServerError writes a 500 status and a JSON encoded error struct +// containing a generic error as the response body (though it will log the given +// one). +func InternalServerError(rw http.ResponseWriter, r *http.Request, err error) { + GetRequestLogger(r).Error(r.Context(), "internal server error", err) + + rw.WriteHeader(500) + JSONResult(rw, r, struct { + Error string `json:"error"` + }{ + Error: "internal server error", + }) +} + +// StrToInt parses the given string as an integer, or returns the given default +// integer if the string is empty. +func StrToInt(str string, defaultVal int) (int, error) { + if str == "" { + return defaultVal, nil + } + return strconv.Atoi(str) +} + +// GetCookie returns the namd cookie's value, or the given default value if the +// cookie is not set. +// +// This will only return an error if there was an unexpected error parsing the +// Request's cookies. +func GetCookie(r *http.Request, cookieName, defaultVal string) (string, error) { + c, err := r.Cookie(cookieName) + if errors.Is(err, http.ErrNoCookie) { + return defaultVal, nil + } else if err != nil { + return "", fmt.Errorf("reading cookie %q: %w", cookieName, err) + } + + return c.Value, nil +} + +// RandStr returns a human-readable random string with the given number of bytes +// of randomness. +func RandStr(numBytes int) string { + b := make([]byte, numBytes) + if _, err := rand.Read(b); err != nil { + panic(err) + } + return hex.EncodeToString(b) +} diff --git a/srv/src/api/chat.go b/srv/src/api/chat.go new file mode 100644 index 0000000..a1acc5a --- /dev/null +++ b/srv/src/api/chat.go @@ -0,0 +1,211 @@ +package api + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "unicode" + + "github.com/gorilla/websocket" + "github.com/mediocregopher/blog.mediocregopher.com/srv/api/apiutils" + "github.com/mediocregopher/blog.mediocregopher.com/srv/chat" +) + +type chatHandler struct { + *http.ServeMux + + room chat.Room + userIDCalc *chat.UserIDCalculator + + wsUpgrader websocket.Upgrader +} + +func newChatHandler( + room chat.Room, userIDCalc *chat.UserIDCalculator, + requirePowMiddleware func(http.Handler) http.Handler, +) http.Handler { + c := &chatHandler{ + ServeMux: http.NewServeMux(), + room: room, + userIDCalc: userIDCalc, + + wsUpgrader: websocket.Upgrader{}, + } + + c.Handle("/history", c.historyHandler()) + c.Handle("/user-id", requirePowMiddleware(c.userIDHandler())) + c.Handle("/append", requirePowMiddleware(c.appendHandler())) + c.Handle("/listen", c.listenHandler()) + + return c +} + +func (c *chatHandler) historyHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + limit, err := apiutils.StrToInt(r.PostFormValue("limit"), 0) + if err != nil { + apiutils.BadRequest(rw, r, fmt.Errorf("invalid limit parameter: %w", err)) + return + } + + cursor := r.PostFormValue("cursor") + + cursor, msgs, err := c.room.History(r.Context(), chat.HistoryOpts{ + Limit: limit, + Cursor: cursor, + }) + + if argErr := (chat.ErrInvalidArg{}); errors.As(err, &argErr) { + apiutils.BadRequest(rw, r, argErr.Err) + return + } else if err != nil { + apiutils.InternalServerError(rw, r, err) + } + + apiutils.JSONResult(rw, r, struct { + Cursor string `json:"cursor"` + Messages []chat.Message `json:"messages"` + }{ + Cursor: cursor, + Messages: msgs, + }) + }) +} + +func (c *chatHandler) userID(r *http.Request) (chat.UserID, error) { + name := r.PostFormValue("name") + if l := len(name); l == 0 { + return chat.UserID{}, errors.New("name is required") + } else if l > 16 { + return chat.UserID{}, errors.New("name too long") + } + + nameClean := strings.Map(func(r rune) rune { + if !unicode.IsPrint(r) { + return -1 + } + return r + }, name) + + if nameClean != name { + return chat.UserID{}, errors.New("name contains invalid characters") + } + + password := r.PostFormValue("password") + if l := len(password); l == 0 { + return chat.UserID{}, errors.New("password is required") + } else if l > 128 { + return chat.UserID{}, errors.New("password too long") + } + + return c.userIDCalc.Calculate(name, password), nil +} + +func (c *chatHandler) userIDHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + userID, err := c.userID(r) + if err != nil { + apiutils.BadRequest(rw, r, err) + return + } + + apiutils.JSONResult(rw, r, struct { + UserID chat.UserID `json:"userID"` + }{ + UserID: userID, + }) + }) +} + +func (c *chatHandler) appendHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + userID, err := c.userID(r) + if err != nil { + apiutils.BadRequest(rw, r, err) + return + } + + body := r.PostFormValue("body") + + if l := len(body); l == 0 { + apiutils.BadRequest(rw, r, errors.New("body is required")) + return + + } else if l > 300 { + apiutils.BadRequest(rw, r, errors.New("body too long")) + return + } + + msg, err := c.room.Append(r.Context(), chat.Message{ + UserID: userID, + Body: body, + }) + + if err != nil { + apiutils.InternalServerError(rw, r, err) + return + } + + apiutils.JSONResult(rw, r, struct { + MessageID string `json:"messageID"` + }{ + MessageID: msg.ID, + }) + }) +} + +func (c *chatHandler) listenHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + + ctx := r.Context() + sinceID := r.FormValue("sinceID") + + conn, err := c.wsUpgrader.Upgrade(rw, r, nil) + if err != nil { + apiutils.BadRequest(rw, r, err) + return + } + defer conn.Close() + + it, err := c.room.Listen(ctx, sinceID) + + if errors.As(err, new(chat.ErrInvalidArg)) { + apiutils.BadRequest(rw, r, err) + return + + } else if errors.Is(err, context.Canceled) { + return + + } else if err != nil { + apiutils.InternalServerError(rw, r, err) + return + } + + defer it.Close() + + for { + + msg, err := it.Next(ctx) + if errors.Is(err, context.Canceled) { + return + + } else if err != nil { + apiutils.InternalServerError(rw, r, err) + return + } + + err = conn.WriteJSON(struct { + Message chat.Message `json:"message"` + }{ + Message: msg, + }) + + if err != nil { + apiutils.GetRequestLogger(r).Error(ctx, "couldn't write message", err) + return + } + } + }) +} diff --git a/srv/src/api/csrf.go b/srv/src/api/csrf.go new file mode 100644 index 0000000..13b6ec6 --- /dev/null +++ b/srv/src/api/csrf.go @@ -0,0 +1,58 @@ +package api + +import ( + "errors" + "net/http" + + "github.com/mediocregopher/blog.mediocregopher.com/srv/api/apiutils" +) + +const ( + csrfTokenCookieName = "csrf_token" + csrfTokenHeaderName = "X-CSRF-Token" +) + +func setCSRFMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + + csrfTok, err := apiutils.GetCookie(r, csrfTokenCookieName, "") + + if err != nil { + apiutils.InternalServerError(rw, r, err) + return + + } else if csrfTok == "" { + http.SetCookie(rw, &http.Cookie{ + Name: csrfTokenCookieName, + Value: apiutils.RandStr(32), + Secure: true, + }) + } + + h.ServeHTTP(rw, r) + }) +} + +func checkCSRFMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + + csrfTok, err := apiutils.GetCookie(r, csrfTokenCookieName, "") + + if err != nil { + apiutils.InternalServerError(rw, r, err) + return + } + + givenCSRFTok := r.Header.Get(csrfTokenHeaderName) + if givenCSRFTok == "" { + givenCSRFTok = r.FormValue("csrfToken") + } + + if csrfTok == "" || givenCSRFTok != csrfTok { + apiutils.BadRequest(rw, r, errors.New("invalid CSRF token")) + return + } + + h.ServeHTTP(rw, r) + }) +} diff --git a/srv/src/api/mailinglist.go b/srv/src/api/mailinglist.go new file mode 100644 index 0000000..d89fe2a --- /dev/null +++ b/srv/src/api/mailinglist.go @@ -0,0 +1,88 @@ +package api + +import ( + "errors" + "net/http" + "strings" + + "github.com/mediocregopher/blog.mediocregopher.com/srv/api/apiutils" + "github.com/mediocregopher/blog.mediocregopher.com/srv/mailinglist" +) + +func (a *api) mailingListSubscribeHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + email := r.PostFormValue("email") + if parts := strings.Split(email, "@"); len(parts) != 2 || + parts[0] == "" || + parts[1] == "" || + len(email) >= 512 { + apiutils.BadRequest(rw, r, errors.New("invalid email")) + return + } + + err := a.params.MailingList.BeginSubscription(email) + + if errors.Is(err, mailinglist.ErrAlreadyVerified) { + // just eat the error, make it look to the user like the + // verification email was sent. + } else if err != nil { + apiutils.InternalServerError(rw, r, err) + return + } + + apiutils.JSONResult(rw, r, struct{}{}) + }) +} + +func (a *api) mailingListFinalizeHandler() http.Handler { + var errInvalidSubToken = errors.New("invalid subToken") + + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + subToken := r.PostFormValue("subToken") + if l := len(subToken); l == 0 || l > 128 { + apiutils.BadRequest(rw, r, errInvalidSubToken) + return + } + + err := a.params.MailingList.FinalizeSubscription(subToken) + + if errors.Is(err, mailinglist.ErrNotFound) { + apiutils.BadRequest(rw, r, errInvalidSubToken) + return + + } else if errors.Is(err, mailinglist.ErrAlreadyVerified) { + // no problem + + } else if err != nil { + apiutils.InternalServerError(rw, r, err) + return + } + + apiutils.JSONResult(rw, r, struct{}{}) + }) +} + +func (a *api) mailingListUnsubscribeHandler() http.Handler { + var errInvalidUnsubToken = errors.New("invalid unsubToken") + + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + unsubToken := r.PostFormValue("unsubToken") + if l := len(unsubToken); l == 0 || l > 128 { + apiutils.BadRequest(rw, r, errInvalidUnsubToken) + return + } + + err := a.params.MailingList.Unsubscribe(unsubToken) + + if errors.Is(err, mailinglist.ErrNotFound) { + apiutils.BadRequest(rw, r, errInvalidUnsubToken) + return + + } else if err != nil { + apiutils.InternalServerError(rw, r, err) + return + } + + apiutils.JSONResult(rw, r, struct{}{}) + }) +} diff --git a/srv/src/api/middleware.go b/srv/src/api/middleware.go new file mode 100644 index 0000000..6ea0d13 --- /dev/null +++ b/srv/src/api/middleware.go @@ -0,0 +1,96 @@ +package api + +import ( + "net" + "net/http" + "time" + + "github.com/mediocregopher/blog.mediocregopher.com/srv/api/apiutils" + "github.com/mediocregopher/mediocre-go-lib/v2/mctx" + "github.com/mediocregopher/mediocre-go-lib/v2/mlog" +) + +func addResponseHeaders(headers map[string]string, h http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + for k, v := range headers { + rw.Header().Set(k, v) + } + h.ServeHTTP(rw, r) + }) +} + +func annotateMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + + type reqInfoKey string + + ip, _, _ := net.SplitHostPort(r.RemoteAddr) + + ctx := r.Context() + ctx = mctx.Annotate(ctx, + reqInfoKey("remote_ip"), ip, + reqInfoKey("url"), r.URL, + reqInfoKey("method"), r.Method, + ) + + r = r.WithContext(ctx) + h.ServeHTTP(rw, r) + }) +} + +type logResponseWriter struct { + http.ResponseWriter + http.Hijacker + statusCode int +} + +func newLogResponseWriter(rw http.ResponseWriter) *logResponseWriter { + h, _ := rw.(http.Hijacker) + return &logResponseWriter{ + ResponseWriter: rw, + Hijacker: h, + statusCode: 200, + } +} + +func (lrw *logResponseWriter) WriteHeader(statusCode int) { + lrw.statusCode = statusCode + lrw.ResponseWriter.WriteHeader(statusCode) +} + +func logMiddleware(logger *mlog.Logger, h http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + + r = apiutils.SetRequestLogger(r, logger) + + lrw := newLogResponseWriter(rw) + + started := time.Now() + h.ServeHTTP(lrw, r) + took := time.Since(started) + + type logCtxKey string + + ctx := r.Context() + ctx = mctx.Annotate(ctx, + logCtxKey("took"), took.String(), + logCtxKey("response_code"), lrw.statusCode, + ) + + logger.Info(ctx, "handled HTTP request") + }) +} + +func postOnlyMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + + // we allow websockets to not be POSTs because, well, they can't be + if r.Method == "POST" || r.Header.Get("Upgrade") == "websocket" { + h.ServeHTTP(rw, r) + return + } + + apiutils.GetRequestLogger(r).WarnString(r.Context(), "method not allowed") + rw.WriteHeader(405) + }) +} diff --git a/srv/src/api/pow.go b/srv/src/api/pow.go new file mode 100644 index 0000000..1b232b1 --- /dev/null +++ b/srv/src/api/pow.go @@ -0,0 +1,53 @@ +package api + +import ( + "encoding/hex" + "errors" + "fmt" + "net/http" + + "github.com/mediocregopher/blog.mediocregopher.com/srv/api/apiutils" +) + +func (a *api) newPowChallengeHandler() http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + + challenge := a.params.PowManager.NewChallenge() + + apiutils.JSONResult(rw, r, struct { + Seed string `json:"seed"` + Target uint32 `json:"target"` + }{ + Seed: hex.EncodeToString(challenge.Seed), + Target: challenge.Target, + }) + }) +} + +func (a *api) requirePowMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + + seedHex := r.FormValue("powSeed") + seed, err := hex.DecodeString(seedHex) + if err != nil || len(seed) == 0 { + apiutils.BadRequest(rw, r, errors.New("invalid powSeed")) + return + } + + solutionHex := r.FormValue("powSolution") + solution, err := hex.DecodeString(solutionHex) + if err != nil || len(seed) == 0 { + apiutils.BadRequest(rw, r, errors.New("invalid powSolution")) + return + } + + err = a.params.PowManager.CheckSolution(seed, solution) + + if err != nil { + apiutils.BadRequest(rw, r, fmt.Errorf("checking proof-of-work solution: %w", err)) + return + } + + h.ServeHTTP(rw, r) + }) +} diff --git a/srv/src/cfg/cfg.go b/srv/src/cfg/cfg.go new file mode 100644 index 0000000..8513e16 --- /dev/null +++ b/srv/src/cfg/cfg.go @@ -0,0 +1,200 @@ +// Package cfg implements a simple wrapper around go's flag package, in order to +// implement initialization hooks. +package cfg + +import ( + "context" + "flag" + "fmt" + "os" + "strconv" + "strings" +) + +// Cfger is a component which can be used with Cfg to setup its initialization. +type Cfger interface { + SetupCfg(*Cfg) +} + +// Params are used to initialize a Cfg instance. +type Params struct { + + // Args are the command line arguments, excluding the command-name. + // + // Defaults to os.Args[1:] + Args []string + + // Env is the process's environment variables. + // + // Defaults to the real environment variables. + Env map[string]string + + // EnvPrefix indicates a string to prefix to all environment variable names + // that Cfg will read. Will be automatically suffixed with a "_" if given. + EnvPrefix string +} + +func (p Params) withDefaults() Params { + + if p.Args == nil { + p.Args = os.Args[1:] + } + + if p.Env == nil { + + p.Env = map[string]string{} + + for _, envVar := range os.Environ() { + + parts := strings.SplitN(envVar, "=", 2) + + if len(parts) < 2 { + panic(fmt.Sprintf("envVar %q returned from os.Environ() somehow", envVar)) + } + + p.Env[parts[0]] = parts[1] + } + } + + if p.EnvPrefix != "" { + p.EnvPrefix = strings.TrimSuffix(p.EnvPrefix, "_") + "_" + } + + return p +} + +// Cfg is a wrapper around the stdlib's FlagSet and a set of initialization +// hooks. +type Cfg struct { + params Params + flagSet *flag.FlagSet + + hooks []func(ctx context.Context) error +} + +// New initializes and returns a new instance of *Cfg. +func New(params Params) *Cfg { + + params = params.withDefaults() + + return &Cfg{ + params: params, + flagSet: flag.NewFlagSet("", flag.ExitOnError), + } +} + +// OnInit appends the given callback to the sequence of hooks which will run on +// a call to Init. +func (c *Cfg) OnInit(cb func(context.Context) error) { + c.hooks = append(c.hooks, cb) +} + +// Init runs all hooks registered using OnInit, in the same order OnInit was +// called. If one returns an error that error is returned and no further hooks +// are run. +func (c *Cfg) Init(ctx context.Context) error { + if err := c.flagSet.Parse(c.params.Args); err != nil { + return err + } + + for _, h := range c.hooks { + if err := h(ctx); err != nil { + return err + } + } + + return nil +} + +func (c *Cfg) envifyName(name string) string { + name = c.params.EnvPrefix + name + name = strings.Replace(name, "-", "_", -1) + name = strings.ToUpper(name) + return name +} + +func envifyUsage(envName, usage string) string { + return fmt.Sprintf("%s (overrides %s)", usage, envName) +} + +// StringVar is equivalent to flag.FlagSet's StringVar method, but will +// additionally set up an environment variable for the parameter. +func (c *Cfg) StringVar(p *string, name, value, usage string) { + + envName := c.envifyName(name) + + c.flagSet.StringVar(p, name, value, envifyUsage(envName, usage)) + + if val := c.params.Env[envName]; val != "" { + *p = val + } +} + +// String is equivalent to flag.FlagSet's String method, but will additionally +// set up an environment variable for the parameter. +func (c *Cfg) String(name, value, usage string) *string { + p := new(string) + c.StringVar(p, name, value, usage) + return p +} + +// IntVar is equivalent to flag.FlagSet's IntVar method, but will additionally +// set up an environment variable for the parameter. +func (c *Cfg) IntVar(p *int, name string, value int, usage string) { + + envName := c.envifyName(name) + + c.flagSet.IntVar(p, name, value, envifyUsage(envName, usage)) + + // if we can't parse the envvar now then just hold onto the error until + // Init, otherwise we'd have to panic here and that'd be ugly. + var err error + + if valStr := c.params.Env[envName]; valStr != "" { + + var val int + val, err = strconv.Atoi(valStr) + + if err != nil { + err = fmt.Errorf( + "parsing envvar %q with value %q: %w", + envName, valStr, err, + ) + + } else { + *p = val + } + } + + c.OnInit(func(context.Context) error { return err }) +} + +// Int is equivalent to flag.FlagSet's Int method, but will additionally set up +// an environment variable for the parameter. +func (c *Cfg) Int(name string, value int, usage string) *int { + p := new(int) + c.IntVar(p, name, value, usage) + return p +} + +// SubCmd should be called _after_ Init. Init will have consumed all arguments +// up until the first non-flag argument. This non-flag argument is a +// sub-command, and is returned by this method. This method also resets Cfg's +// internal state so that new options can be added to it. +// +// If there is no sub-command following the initial set of flags then this will +// return empty string. +func (c *Cfg) SubCmd() string { + c.params.Args = c.flagSet.Args() + if len(c.params.Args) == 0 { + return "" + } + + subCmd := c.params.Args[0] + + c.flagSet = flag.NewFlagSet(subCmd, flag.ExitOnError) + c.hooks = nil + c.params.Args = c.params.Args[1:] + + return subCmd +} diff --git a/srv/src/cfg/cfg_test.go b/srv/src/cfg/cfg_test.go new file mode 100644 index 0000000..7ccf94a --- /dev/null +++ b/srv/src/cfg/cfg_test.go @@ -0,0 +1,46 @@ +package cfg + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringVar(t *testing.T) { + + cfg := New(Params{ + Args: []string{"--foo=CLI"}, + Env: map[string]string{"FOO": "ENV", "BAR": "ENV"}, + }) + + var foo, bar, baz string + + cfg.StringVar(&foo, "foo", "DEF", "") + cfg.StringVar(&bar, "bar", "DEF", "") + cfg.StringVar(&baz, "baz", "DEF", "") + + assert.NoError(t, cfg.Init(context.Background())) + assert.Equal(t, "CLI", foo) + assert.Equal(t, "ENV", bar) + assert.Equal(t, "DEF", baz) +} + +func TestIntVar(t *testing.T) { + + cfg := New(Params{ + Args: []string{"--foo=111"}, + Env: map[string]string{"FOO": "222", "BAR": "222"}, + }) + + var foo, bar, baz int + + cfg.IntVar(&foo, "foo", 333, "") + cfg.IntVar(&bar, "bar", 333, "") + cfg.IntVar(&baz, "baz", 333, "") + + assert.NoError(t, cfg.Init(context.Background())) + assert.Equal(t, 111, foo) + assert.Equal(t, 222, bar) + assert.Equal(t, 333, baz) +} diff --git a/srv/src/chat/chat.go b/srv/src/chat/chat.go new file mode 100644 index 0000000..0a88d3b --- /dev/null +++ b/srv/src/chat/chat.go @@ -0,0 +1,467 @@ +// Package chat implements a simple chatroom system. +package chat + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "sync" + "time" + + "github.com/mediocregopher/mediocre-go-lib/v2/mctx" + "github.com/mediocregopher/mediocre-go-lib/v2/mlog" + "github.com/mediocregopher/radix/v4" +) + +// ErrInvalidArg is returned from methods in this package when a call fails due +// to invalid input. +type ErrInvalidArg struct { + Err error +} + +func (e ErrInvalidArg) Error() string { + return fmt.Sprintf("invalid argument: %v", e.Err) +} + +var ( + errInvalidMessageID = ErrInvalidArg{Err: errors.New("invalid Message ID")} +) + +// Message describes a message which has been posted to a Room. +type Message struct { + ID string `json:"id"` + UserID UserID `json:"userID"` + Body string `json:"body"` + CreatedAt int64 `json:"createdAt,omitempty"` +} + +func msgFromStreamEntry(entry radix.StreamEntry) (Message, error) { + + // NOTE this should probably be a shortcut in radix + var bodyStr string + for _, field := range entry.Fields { + if field[0] == "json" { + bodyStr = field[1] + break + } + } + + if bodyStr == "" { + return Message{}, errors.New("no 'json' field") + } + + var msg Message + if err := json.Unmarshal([]byte(bodyStr), &msg); err != nil { + return Message{}, fmt.Errorf( + "json unmarshaling body %q: %w", bodyStr, err, + ) + } + + msg.ID = entry.ID.String() + msg.CreatedAt = int64(entry.ID.Time / 1000) + return msg, nil +} + +// MessageIterator returns a sequence of Messages which may or may not be +// unbounded. +type MessageIterator interface { + + // Next blocks until it returns the next Message in the sequence, or the + // context error if the context is cancelled, or io.EOF if the sequence has + // been exhausted. + Next(context.Context) (Message, error) + + // Close should always be called once Next has returned an error or the + // MessageIterator will no longer be used. + Close() error +} + +// HistoryOpts are passed into Room's History method in order to affect its +// result. All fields are optional. +type HistoryOpts struct { + Limit int // defaults to, and is capped at, 100. + Cursor string // If not given then the most recent Messages are returned. +} + +func (o HistoryOpts) sanitize() (HistoryOpts, error) { + if o.Limit <= 0 || o.Limit > 100 { + o.Limit = 100 + } + + if o.Cursor != "" { + id, err := parseStreamEntryID(o.Cursor) + if err != nil { + return HistoryOpts{}, fmt.Errorf("parsing Cursor: %w", err) + } + o.Cursor = id.String() + } + + return o, nil +} + +// Room implements functionality related to a single, unique chat room. +type Room interface { + + // Append accepts a new Message and stores it at the end of the room's + // history. The original Message is returned with any relevant fields (e.g. + // ID) updated. + Append(context.Context, Message) (Message, error) + + // Returns a cursor and the list of historical Messages in time descending + // order. The cursor can be passed into the next call to History to receive + // the next set of Messages. + History(context.Context, HistoryOpts) (string, []Message, error) + + // Listen returns a MessageIterator which will return all Messages appended + // to the Room since the given ID. Once all existing messages are iterated + // through then the MessageIterator will begin blocking until a new Message + // is posted. + Listen(ctx context.Context, sinceID string) (MessageIterator, error) + + // Delete deletes a Message from the Room. + Delete(ctx context.Context, id string) error + + // Close is used to clean up all resources created by the Room. + Close() error +} + +// RoomParams are used to instantiate a new Room. All fields are required unless +// otherwise noted. +type RoomParams struct { + Logger *mlog.Logger + Redis radix.Client + ID string + MaxMessages int +} + +func (p RoomParams) streamKey() string { + return fmt.Sprintf("chat:{%s}:stream", p.ID) +} + +type room struct { + params RoomParams + + closeCtx context.Context + closeCancel context.CancelFunc + wg sync.WaitGroup + + listeningL sync.Mutex + listening map[chan Message]struct{} + listeningLastID radix.StreamEntryID +} + +// NewRoom initializes and returns a new Room instance. +func NewRoom(ctx context.Context, params RoomParams) (Room, error) { + + params.Logger = params.Logger.WithNamespace("chat-room") + + r := &room{ + params: params, + listening: map[chan Message]struct{}{}, + } + + r.closeCtx, r.closeCancel = context.WithCancel(context.Background()) + + // figure out the most recent message, if any. + lastEntryID, err := r.mostRecentMsgID(ctx) + if err != nil { + return nil, fmt.Errorf("discovering most recent entry ID in stream: %w", err) + } + r.listeningLastID = lastEntryID + + r.wg.Add(1) + go func() { + defer r.wg.Done() + r.readStreamLoop(r.closeCtx) + }() + + return r, nil +} + +func (r *room) Close() error { + r.closeCancel() + r.wg.Wait() + return nil +} + +func (r *room) mostRecentMsgID(ctx context.Context) (radix.StreamEntryID, error) { + + var entries []radix.StreamEntry + err := r.params.Redis.Do(ctx, radix.Cmd( + &entries, + "XREVRANGE", r.params.streamKey(), "+", "-", "COUNT", "1", + )) + + if err != nil || len(entries) == 0 { + return radix.StreamEntryID{}, err + } + + return entries[0].ID, nil +} + +func (r *room) Append(ctx context.Context, msg Message) (Message, error) { + msg.ID = "" // just in case + + b, err := json.Marshal(msg) + if err != nil { + return Message{}, fmt.Errorf("json marshaling Message: %w", err) + } + + key := r.params.streamKey() + maxLen := strconv.Itoa(r.params.MaxMessages) + body := string(b) + + var id radix.StreamEntryID + + err = r.params.Redis.Do(ctx, radix.Cmd( + &id, "XADD", key, "MAXLEN", "=", maxLen, "*", "json", body, + )) + + if err != nil { + return Message{}, fmt.Errorf("posting message to redis: %w", err) + } + + msg.ID = id.String() + msg.CreatedAt = int64(id.Time / 1000) + return msg, nil +} + +const zeroCursor = "0-0" + +func (r *room) History(ctx context.Context, opts HistoryOpts) (string, []Message, error) { + opts, err := opts.sanitize() + if err != nil { + return "", nil, err + } + + key := r.params.streamKey() + end := opts.Cursor + if end == "" { + end = "+" + } + start := "-" + count := strconv.Itoa(opts.Limit) + + msgs := make([]Message, 0, opts.Limit) + streamEntries := make([]radix.StreamEntry, 0, opts.Limit) + + err = r.params.Redis.Do(ctx, radix.Cmd( + &streamEntries, + "XREVRANGE", key, end, start, "COUNT", count, + )) + + if err != nil { + return "", nil, fmt.Errorf("calling XREVRANGE: %w", err) + } + + var oldestEntryID radix.StreamEntryID + + for _, entry := range streamEntries { + oldestEntryID = entry.ID + + msg, err := msgFromStreamEntry(entry) + if err != nil { + return "", nil, fmt.Errorf( + "parsing stream entry %q: %w", entry.ID, err, + ) + } + msgs = append(msgs, msg) + } + + if len(msgs) < opts.Limit { + return zeroCursor, msgs, nil + } + + cursor := oldestEntryID.Prev() + return cursor.String(), msgs, nil +} + +func (r *room) readStream(ctx context.Context) error { + + r.listeningL.Lock() + lastEntryID := r.listeningLastID + r.listeningL.Unlock() + + redisAddr := r.params.Redis.Addr() + redisConn, err := radix.Dial(ctx, redisAddr.Network(), redisAddr.String()) + if err != nil { + return fmt.Errorf("creating redis connection: %w", err) + } + defer redisConn.Close() + + streamReader := (radix.StreamReaderConfig{}).New( + redisConn, + map[string]radix.StreamConfig{ + r.params.streamKey(): {After: lastEntryID}, + }, + ) + + for { + dlCtx, dlCtxCancel := context.WithTimeout(ctx, 10*time.Second) + _, streamEntry, err := streamReader.Next(dlCtx) + dlCtxCancel() + + if errors.Is(err, radix.ErrNoStreamEntries) { + continue + } else if err != nil { + return fmt.Errorf("fetching next entry from stream: %w", err) + } + + msg, err := msgFromStreamEntry(streamEntry) + if err != nil { + return fmt.Errorf("parsing stream entry %q: %w", streamEntry, err) + } + + r.listeningL.Lock() + + var dropped int + for ch := range r.listening { + select { + case ch <- msg: + default: + dropped++ + } + } + + if dropped > 0 { + ctx := mctx.Annotate(ctx, "msgID", msg.ID, "dropped", dropped) + r.params.Logger.WarnString(ctx, "some listening channels full, messages dropped") + } + + r.listeningLastID = streamEntry.ID + + r.listeningL.Unlock() + } +} + +func (r *room) readStreamLoop(ctx context.Context) { + for { + err := r.readStream(ctx) + if errors.Is(err, context.Canceled) { + return + } else if err != nil { + r.params.Logger.Error(ctx, "reading from redis stream", err) + } + } +} + +type listenMsgIterator struct { + ch <-chan Message + missedMsgs []Message + sinceEntryID radix.StreamEntryID + cleanup func() +} + +func (i *listenMsgIterator) Next(ctx context.Context) (Message, error) { + + if len(i.missedMsgs) > 0 { + msg := i.missedMsgs[0] + i.missedMsgs = i.missedMsgs[1:] + return msg, nil + } + + for { + select { + case <-ctx.Done(): + return Message{}, ctx.Err() + case msg := <-i.ch: + + entryID, err := parseStreamEntryID(msg.ID) + if err != nil { + return Message{}, fmt.Errorf("parsing Message ID %q: %w", msg.ID, err) + + } else if !i.sinceEntryID.Before(entryID) { + // this can happen if someone Appends a Message at the same time + // as another calls Listen. The Listener might have already seen + // the Message by calling History prior to the stream reader + // having processed it and updating listeningLastID. + continue + } + + return msg, nil + } + } +} + +func (i *listenMsgIterator) Close() error { + i.cleanup() + return nil +} + +func (r *room) Listen( + ctx context.Context, sinceID string, +) ( + MessageIterator, error, +) { + + var sinceEntryID radix.StreamEntryID + + if sinceID != "" { + var err error + if sinceEntryID, err = parseStreamEntryID(sinceID); err != nil { + return nil, fmt.Errorf("parsing sinceID: %w", err) + } + } + + ch := make(chan Message, 32) + + r.listeningL.Lock() + lastEntryID := r.listeningLastID + r.listening[ch] = struct{}{} + r.listeningL.Unlock() + + cleanup := func() { + r.listeningL.Lock() + defer r.listeningL.Unlock() + delete(r.listening, ch) + } + + key := r.params.streamKey() + start := sinceEntryID.Next().String() + end := "+" + if lastEntryID != (radix.StreamEntryID{}) { + end = lastEntryID.String() + } + + var streamEntries []radix.StreamEntry + + err := r.params.Redis.Do(ctx, radix.Cmd( + &streamEntries, + "XRANGE", key, start, end, + )) + + if err != nil { + cleanup() + return nil, fmt.Errorf("retrieving missed stream entries: %w", err) + } + + missedMsgs := make([]Message, len(streamEntries)) + + for i := range streamEntries { + + msg, err := msgFromStreamEntry(streamEntries[i]) + if err != nil { + cleanup() + return nil, fmt.Errorf( + "parsing stream entry %q: %w", streamEntries[i].ID, err, + ) + } + + missedMsgs[i] = msg + } + + return &listenMsgIterator{ + ch: ch, + missedMsgs: missedMsgs, + sinceEntryID: sinceEntryID, + cleanup: cleanup, + }, nil +} + +func (r *room) Delete(ctx context.Context, id string) error { + return r.params.Redis.Do(ctx, radix.Cmd( + nil, "XDEL", r.params.streamKey(), id, + )) +} diff --git a/srv/src/chat/chat_test.go b/srv/src/chat/chat_test.go new file mode 100644 index 0000000..d37921c --- /dev/null +++ b/srv/src/chat/chat_test.go @@ -0,0 +1,200 @@ +package chat + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/google/uuid" + "github.com/mediocregopher/mediocre-go-lib/v2/mlog" + "github.com/mediocregopher/radix/v4" + "github.com/stretchr/testify/assert" +) + +const roomTestHarnessMaxMsgs = 10 + +type roomTestHarness struct { + ctx context.Context + room Room + allMsgs []Message +} + +func (h *roomTestHarness) newMsg(t *testing.T) Message { + msg, err := h.room.Append(h.ctx, Message{ + UserID: UserID{ + Name: uuid.New().String(), + Hash: "0000", + }, + Body: uuid.New().String(), + }) + assert.NoError(t, err) + + t.Logf("appended message %s", msg.ID) + + h.allMsgs = append([]Message{msg}, h.allMsgs...) + + if len(h.allMsgs) > roomTestHarnessMaxMsgs { + h.allMsgs = h.allMsgs[:roomTestHarnessMaxMsgs] + } + + return msg +} + +func newRoomTestHarness(t *testing.T) *roomTestHarness { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + + redis, err := radix.Dial(ctx, "tcp", "127.0.0.1:6379") + assert.NoError(t, err) + t.Cleanup(func() { redis.Close() }) + + roomParams := RoomParams{ + Logger: mlog.NewLogger(nil), + Redis: redis, + ID: uuid.New().String(), + MaxMessages: roomTestHarnessMaxMsgs, + } + + t.Logf("creating test Room %q", roomParams.ID) + room, err := NewRoom(ctx, roomParams) + assert.NoError(t, err) + + t.Cleanup(func() { + err := redis.Do(context.Background(), radix.Cmd( + nil, "DEL", roomParams.streamKey(), + )) + assert.NoError(t, err) + }) + + return &roomTestHarness{ctx: ctx, room: room} +} + +func TestRoom(t *testing.T) { + t.Run("history", func(t *testing.T) { + + tests := []struct { + numMsgs int + limit int + }{ + {numMsgs: 0, limit: 1}, + {numMsgs: 1, limit: 1}, + {numMsgs: 2, limit: 1}, + {numMsgs: 2, limit: 10}, + {numMsgs: 9, limit: 2}, + {numMsgs: 9, limit: 3}, + {numMsgs: 9, limit: 4}, + {numMsgs: 15, limit: 3}, + } + + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Logf("test: %+v", test) + + h := newRoomTestHarness(t) + + for j := 0; j < test.numMsgs; j++ { + h.newMsg(t) + } + + var gotMsgs []Message + var cursor string + + for { + + var msgs []Message + var err error + cursor, msgs, err = h.room.History(h.ctx, HistoryOpts{ + Cursor: cursor, + Limit: test.limit, + }) + + assert.NoError(t, err) + assert.NotEmpty(t, cursor) + + if len(msgs) == 0 { + break + } + + gotMsgs = append(gotMsgs, msgs...) + } + + assert.Equal(t, h.allMsgs, gotMsgs) + }) + } + }) + + assertNextMsg := func( + t *testing.T, expMsg Message, + ctx context.Context, it MessageIterator, + ) { + t.Helper() + gotMsg, err := it.Next(ctx) + assert.NoError(t, err) + assert.Equal(t, expMsg, gotMsg) + } + + t.Run("listen/already_populated", func(t *testing.T) { + h := newRoomTestHarness(t) + + msgA, msgB, msgC := h.newMsg(t), h.newMsg(t), h.newMsg(t) + _ = msgA + _ = msgB + + itFoo, err := h.room.Listen(h.ctx, msgC.ID) + assert.NoError(t, err) + defer itFoo.Close() + + itBar, err := h.room.Listen(h.ctx, msgA.ID) + assert.NoError(t, err) + defer itBar.Close() + + msgD := h.newMsg(t) + + // itBar should get msgB and msgC before anything else. + assertNextMsg(t, msgB, h.ctx, itBar) + assertNextMsg(t, msgC, h.ctx, itBar) + + // now both iterators should give msgD + assertNextMsg(t, msgD, h.ctx, itFoo) + assertNextMsg(t, msgD, h.ctx, itBar) + + // timeout should be honored + { + timeoutCtx, timeoutCancel := context.WithTimeout(h.ctx, 1*time.Second) + _, errFoo := itFoo.Next(timeoutCtx) + _, errBar := itBar.Next(timeoutCtx) + timeoutCancel() + + assert.ErrorIs(t, errFoo, context.DeadlineExceeded) + assert.ErrorIs(t, errBar, context.DeadlineExceeded) + } + + // new message should work + { + expMsg := h.newMsg(t) + + timeoutCtx, timeoutCancel := context.WithTimeout(h.ctx, 1*time.Second) + gotFooMsg, errFoo := itFoo.Next(timeoutCtx) + gotBarMsg, errBar := itBar.Next(timeoutCtx) + timeoutCancel() + + assert.Equal(t, expMsg, gotFooMsg) + assert.NoError(t, errFoo) + assert.Equal(t, expMsg, gotBarMsg) + assert.NoError(t, errBar) + } + + }) + + t.Run("listen/empty", func(t *testing.T) { + h := newRoomTestHarness(t) + + it, err := h.room.Listen(h.ctx, "") + assert.NoError(t, err) + defer it.Close() + + msg := h.newMsg(t) + assertNextMsg(t, msg, h.ctx, it) + }) +} diff --git a/srv/src/chat/user.go b/srv/src/chat/user.go new file mode 100644 index 0000000..3f5ab95 --- /dev/null +++ b/srv/src/chat/user.go @@ -0,0 +1,68 @@ +package chat + +import ( + "encoding/hex" + "fmt" + "sync" + + "golang.org/x/crypto/argon2" +) + +// UserID uniquely identifies an individual user who has posted a message in a +// Room. +type UserID struct { + + // Name will be the user's chosen display name. + Name string `json:"name"` + + // Hash will be a hex string generated from a secret only the user knows. + Hash string `json:"id"` +} + +// UserIDCalculator is used to calculate UserIDs. +type UserIDCalculator struct { + + // Secret is used when calculating UserID Hash salts. + Secret []byte + + // TimeCost, MemoryCost, and Threads are used as inputs to the Argon2id + // algorithm which is used to generate the Hash. + TimeCost, MemoryCost uint32 + Threads uint8 + + // HashLen specifies the number of bytes the Hash should be. + HashLen uint32 + + // Lock, if set, forces concurrent Calculate calls to occur sequentially. + Lock *sync.Mutex +} + +// NewUserIDCalculator returns a UserIDCalculator with sane defaults. +func NewUserIDCalculator(secret []byte) *UserIDCalculator { + return &UserIDCalculator{ + Secret: secret, + TimeCost: 15, + MemoryCost: 128 * 1024, + Threads: 2, + HashLen: 16, + Lock: new(sync.Mutex), + } +} + +// Calculate accepts a name and password and returns the calculated UserID. +func (c *UserIDCalculator) Calculate(name, password string) UserID { + + input := fmt.Sprintf("%q:%q", name, password) + + hashB := argon2.IDKey( + []byte(input), + c.Secret, // salt + c.TimeCost, c.MemoryCost, c.Threads, + c.HashLen, + ) + + return UserID{ + Name: name, + Hash: hex.EncodeToString(hashB), + } +} diff --git a/srv/src/chat/user_test.go b/srv/src/chat/user_test.go new file mode 100644 index 0000000..2169cde --- /dev/null +++ b/srv/src/chat/user_test.go @@ -0,0 +1,26 @@ +package chat + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUserIDCalculator(t *testing.T) { + + const name, password = "name", "password" + + c := NewUserIDCalculator([]byte("foo")) + + // calculating with same params twice should result in same UserID + userID := c.Calculate(name, password) + assert.Equal(t, userID, c.Calculate(name, password)) + + // changing either name or password should result in a different Hash + assert.NotEqual(t, userID.Hash, c.Calculate(name+"!", password).Hash) + assert.NotEqual(t, userID.Hash, c.Calculate(name, password+"!").Hash) + + // changing the secret should change the UserID + c.Secret = []byte("bar") + assert.NotEqual(t, userID, c.Calculate(name, password)) +} diff --git a/srv/src/chat/util.go b/srv/src/chat/util.go new file mode 100644 index 0000000..05f4830 --- /dev/null +++ b/srv/src/chat/util.go @@ -0,0 +1,28 @@ +package chat + +import ( + "strconv" + "strings" + + "github.com/mediocregopher/radix/v4" +) + +func parseStreamEntryID(str string) (radix.StreamEntryID, error) { + + split := strings.SplitN(str, "-", 2) + if len(split) != 2 { + return radix.StreamEntryID{}, errInvalidMessageID + } + + time, err := strconv.ParseUint(split[0], 10, 64) + if err != nil { + return radix.StreamEntryID{}, errInvalidMessageID + } + + seq, err := strconv.ParseUint(split[1], 10, 64) + if err != nil { + return radix.StreamEntryID{}, errInvalidMessageID + } + + return radix.StreamEntryID{Time: time, Seq: seq}, nil +} diff --git a/srv/src/cmd/mailinglist-cli/main.go b/srv/src/cmd/mailinglist-cli/main.go new file mode 100644 index 0000000..c3207df --- /dev/null +++ b/srv/src/cmd/mailinglist-cli/main.go @@ -0,0 +1,120 @@ +package main + +import ( + "context" + "errors" + "io" + "path" + + "github.com/mediocregopher/blog.mediocregopher.com/srv/cfg" + "github.com/mediocregopher/blog.mediocregopher.com/srv/mailinglist" + "github.com/mediocregopher/mediocre-go-lib/v2/mctx" + "github.com/mediocregopher/mediocre-go-lib/v2/mlog" + "github.com/tilinna/clock" +) + +func main() { + + ctx := context.Background() + + cfg := cfg.New(cfg.Params{ + EnvPrefix: "MEDIOCRE_BLOG", + }) + + dataDir := cfg.String("data-dir", ".", "Directory to use for long term storage") + + var mailerParams mailinglist.MailerParams + mailerParams.SetupCfg(cfg) + ctx = mctx.WithAnnotator(ctx, &mailerParams) + + var mlParams mailinglist.Params + mlParams.SetupCfg(cfg) + ctx = mctx.WithAnnotator(ctx, &mlParams) + + // initialization + err := cfg.Init(ctx) + + logger := mlog.NewLogger(nil) + defer logger.Close() + + logger.Info(ctx, "process started") + defer logger.Info(ctx, "process exiting") + + if err != nil { + logger.Fatal(ctx, "initializing", err) + } + + clock := clock.Realtime() + + var mailer mailinglist.Mailer + if mailerParams.SMTPAddr == "" { + logger.Info(ctx, "-smtp-addr not given, using NullMailer") + mailer = mailinglist.NullMailer + } else { + mailer = mailinglist.NewMailer(mailerParams) + } + + mailingListDBFile := path.Join(*dataDir, "mailinglist.sqlite3") + ctx = mctx.Annotate(ctx, "mailingListDBFile", mailingListDBFile) + + mlStore, err := mailinglist.NewStore(mailingListDBFile) + if err != nil { + logger.Fatal(ctx, "initializing mailing list storage", err) + } + defer mlStore.Close() + + mlParams.Store = mlStore + mlParams.Mailer = mailer + mlParams.Clock = clock + + ml := mailinglist.New(mlParams) + + subCmd := cfg.SubCmd() + ctx = mctx.Annotate(ctx, "subCmd", subCmd) + + switch subCmd { + + case "list": + + for it := mlStore.GetAll(); ; { + email, err := it() + if errors.Is(err, io.EOF) { + break + } else if err != nil { + logger.Fatal(ctx, "retrieving next email", err) + } + + ctx := mctx.Annotate(context.Background(), + "email", email.Email, + "createdAt", email.CreatedAt, + "verifiedAt", email.VerifiedAt, + ) + + logger.Info(ctx, "next") + } + + case "publish": + + title := cfg.String("title", "", "Title of the post which was published") + url := cfg.String("url", "", "URL of the post which was published") + + if err := cfg.Init(ctx); err != nil { + logger.Fatal(ctx, "initializing", err) + } + + if *title == "" { + logger.FatalString(ctx, "-title is required") + + } else if *url == "" { + logger.FatalString(ctx, "-url is required") + } + + err := ml.Publish(*title, *url) + if err != nil { + logger.Fatal(ctx, "publishing", err) + } + + default: + logger.FatalString(ctx, "invalid sub-command, must be list|publish") + } +} diff --git a/srv/src/cmd/mediocre-blog/main.go b/srv/src/cmd/mediocre-blog/main.go new file mode 100644 index 0000000..4cf3024 --- /dev/null +++ b/srv/src/cmd/mediocre-blog/main.go @@ -0,0 +1,160 @@ +package main + +import ( + "context" + "os" + "os/signal" + "path" + "syscall" + "time" + + "github.com/mediocregopher/blog.mediocregopher.com/srv/api" + "github.com/mediocregopher/blog.mediocregopher.com/srv/cfg" + "github.com/mediocregopher/blog.mediocregopher.com/srv/chat" + "github.com/mediocregopher/blog.mediocregopher.com/srv/mailinglist" + "github.com/mediocregopher/blog.mediocregopher.com/srv/pow" + "github.com/mediocregopher/mediocre-go-lib/v2/mctx" + "github.com/mediocregopher/mediocre-go-lib/v2/mlog" + "github.com/mediocregopher/radix/v4" + "github.com/tilinna/clock" +) + +func main() { + + ctx := context.Background() + + cfg := cfg.New(cfg.Params{ + EnvPrefix: "MEDIOCRE_BLOG", + }) + + dataDir := cfg.String("data-dir", ".", "Directory to use for long term storage") + + var powMgrParams pow.ManagerParams + powMgrParams.SetupCfg(cfg) + ctx = mctx.WithAnnotator(ctx, &powMgrParams) + + var mailerParams mailinglist.MailerParams + mailerParams.SetupCfg(cfg) + ctx = mctx.WithAnnotator(ctx, &mailerParams) + + var mlParams mailinglist.Params + mlParams.SetupCfg(cfg) + ctx = mctx.WithAnnotator(ctx, &mlParams) + + var apiParams api.Params + apiParams.SetupCfg(cfg) + ctx = mctx.WithAnnotator(ctx, &apiParams) + + redisProto := cfg.String("redis-proto", "tcp", "Network protocol to connect to redis over, can be tcp or unix") + redisAddr := cfg.String("redis-addr", "127.0.0.1:6379", "Address redis is expected to listen on") + redisPoolSize := cfg.Int("redis-pool-size", 5, "Number of connections in the redis pool to keep") + + chatGlobalRoomMaxMsgs := cfg.Int("chat-global-room-max-messages", 1000, "Maximum number of messages the global chat room can retain") + chatUserIDCalcSecret := cfg.String("chat-user-id-calc-secret", "", "Secret to use when calculating user ids") + + // initialization + err := cfg.Init(ctx) + + logger := mlog.NewLogger(nil) + defer logger.Close() + + logger.Info(ctx, "process started") + defer logger.Info(ctx, "process exiting") + + if err != nil { + logger.Fatal(ctx, "initializing", err) + } + + ctx = mctx.Annotate(ctx, + "dataDir", *dataDir, + "redisProto", *redisProto, + "redisAddr", *redisAddr, + "redisPoolSize", *redisPoolSize, + "chatGlobalRoomMaxMsgs", *chatGlobalRoomMaxMsgs, + ) + + clock := clock.Realtime() + + powStore := pow.NewMemoryStore(clock) + defer powStore.Close() + + powMgrParams.Store = powStore + powMgrParams.Clock = clock + + powMgr := pow.NewManager(powMgrParams) + + var mailer mailinglist.Mailer + if mailerParams.SMTPAddr == "" { + logger.Info(ctx, "-smtp-addr not given, using a fake Mailer") + mailer = mailinglist.NewLogMailer(logger.WithNamespace("fake-mailer")) + } else { + mailer = mailinglist.NewMailer(mailerParams) + } + + mailingListDBFile := path.Join(*dataDir, "mailinglist.sqlite3") + ctx = mctx.Annotate(ctx, "mailingListDBFile", mailingListDBFile) + + mlStore, err := mailinglist.NewStore(mailingListDBFile) + if err != nil { + logger.Fatal(ctx, "initializing mailing list storage", err) + } + defer mlStore.Close() + + mlParams.Store = mlStore + mlParams.Mailer = mailer + mlParams.Clock = clock + + ml := mailinglist.New(mlParams) + + redis, err := (radix.PoolConfig{ + Size: *redisPoolSize, + }).New( + ctx, *redisProto, *redisAddr, + ) + + if err != nil { + logger.Fatal(ctx, "initializing redis pool", err) + } + defer redis.Close() + + chatGlobalRoom, err := chat.NewRoom(ctx, chat.RoomParams{ + Logger: logger.WithNamespace("global-chat-room"), + Redis: redis, + ID: "global", + MaxMessages: *chatGlobalRoomMaxMsgs, + }) + if err != nil { + logger.Fatal(ctx, "initializing global chat room", err) + } + defer chatGlobalRoom.Close() + + chatUserIDCalc := chat.NewUserIDCalculator([]byte(*chatUserIDCalcSecret)) + + apiParams.Logger = logger.WithNamespace("api") + apiParams.PowManager = powMgr + apiParams.MailingList = ml + apiParams.GlobalRoom = chatGlobalRoom + apiParams.UserIDCalculator = chatUserIDCalc + + logger.Info(ctx, "listening") + a, err := api.New(apiParams) + if err != nil { + logger.Fatal(ctx, "initializing api", err) + } + defer func() { + shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + if err := a.Shutdown(shutdownCtx); err != nil { + logger.Fatal(ctx, "shutting down api", err) + } + }() + + // wait + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + // let the defers begin +} diff --git a/srv/src/cmd/userid-calc-cli/main.go b/srv/src/cmd/userid-calc-cli/main.go new file mode 100644 index 0000000..90c44e7 --- /dev/null +++ b/srv/src/cmd/userid-calc-cli/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + + "github.com/mediocregopher/blog.mediocregopher.com/srv/chat" +) + +func main() { + + secret := flag.String("secret", "", "Secret to use when calculating UserIDs") + name := flag.String("name", "", "") + password := flag.String("password", "", "") + + flag.Parse() + + calc := chat.NewUserIDCalculator([]byte(*secret)) + userID := calc.Calculate(*name, *password) + + b, err := json.Marshal(userID) + if err != nil { + panic(err) + } + + fmt.Println(string(b)) +} diff --git a/srv/src/go.mod b/srv/src/go.mod new file mode 100644 index 0000000..6a912e2 --- /dev/null +++ b/srv/src/go.mod @@ -0,0 +1,18 @@ +module github.com/mediocregopher/blog.mediocregopher.com/srv + +go 1.16 + +require ( + github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 + github.com/emersion/go-smtp v0.15.0 + github.com/google/uuid v1.3.0 + github.com/gorilla/websocket v1.4.2 // indirect + github.com/mattn/go-sqlite3 v1.14.8 + github.com/mediocregopher/mediocre-go-lib/v2 v2.0.0-beta.0.0.20220506011745-cbeee71cb1ee + github.com/mediocregopher/radix/v4 v4.0.0-beta.1.0.20210726230805-d62fa1b2e3cb // indirect + github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc + github.com/stretchr/testify v1.7.0 + github.com/tilinna/clock v1.1.0 + github.com/ziutek/mymysql v1.5.4 // indirect + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect +) diff --git a/srv/src/go.sum b/srv/src/go.sum new file mode 100644 index 0000000..77aac2e --- /dev/null +++ b/srv/src/go.sum @@ -0,0 +1,247 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-smtp v0.15.0 h1:3+hMGMGrqP/lqd7qoxZc1hTU8LY8gHV9RFGWlqSDmP8= +github.com/emersion/go-smtp v0.15.0/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/logger v1.0.3 h1:YaXOTHNPCvkqqA7w05A4v0k2tCdpr+sgFlgINbQ6gqc= +github.com/gobuffalo/logger v1.0.3/go.mod h1:SoeejUwldiS7ZsyCBphOGURmWdwUFXs0J7TCjEhjKxM= +github.com/gobuffalo/packd v1.0.0 h1:6ERZvJHfe24rfFmA9OaoKBdC7+c9sydrytMg8SdFGBM= +github.com/gobuffalo/packd v1.0.0/go.mod h1:6VTc4htmJRFB7u1m/4LeMTWjFoYrUiBkU9Fdec9hrhI= +github.com/gobuffalo/packr/v2 v2.8.1 h1:tkQpju6i3EtMXJ9uoF5GT6kB+LMTimDWD8Xvbz6zDVA= +github.com/gobuffalo/packr/v2 v2.8.1/go.mod h1:c/PLlOuTU+p3SybaJATW3H6lX/iK7xEz5OeMf+NnJpg= +github.com/godror/godror v0.24.2/go.mod h1:wZv/9vPiUib6tkoDl+AZ/QLf5YZgMravZ7jxH2eQWAE= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karrick/godirwalk v1.15.8 h1:7+rWAZPn9zuRxaIqqT8Ohs2Q2Ac0msBqwRdxNCr2VVs= +github.com/karrick/godirwalk v1.15.8/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kortschak/utter v1.0.1/go.mod h1:vSmSjbyrlKjjsL71193LmzBOKgwePk9DH6uFaWHIInc= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= +github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= +github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= +github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= +github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= +github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-oci8 v0.1.1/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU= +github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mediocregopher/mediocre-go-lib/v2 v2.0.0-beta.0 h1:i9FBkcCaWXxteJ8458AD8dBL2YqSxVlpsHOMWg5N9Dc= +github.com/mediocregopher/mediocre-go-lib/v2 v2.0.0-beta.0/go.mod h1:wOZVlnKYvIbkzyCJ3dxy1k40XkirvCd1pisX2O91qoQ= +github.com/mediocregopher/mediocre-go-lib/v2 v2.0.0-beta.0.0.20220506011745-cbeee71cb1ee h1:AWRuhgn7iumyhPuxKwed1F1Ri2dXMwxKfp5YIdpnQIY= +github.com/mediocregopher/mediocre-go-lib/v2 v2.0.0-beta.0.0.20220506011745-cbeee71cb1ee/go.mod h1:wOZVlnKYvIbkzyCJ3dxy1k40XkirvCd1pisX2O91qoQ= +github.com/mediocregopher/radix/v4 v4.0.0-beta.1.0.20210726230805-d62fa1b2e3cb h1:7Y2vAC5q44VJzbBUdxRUEqfz88ySJ/6yXXkpQ+sxke4= +github.com/mediocregopher/radix/v4 v4.0.0-beta.1.0.20210726230805-d62fa1b2e3cb/go.mod h1:ajchozX/6ELmydxWeWM6xCFHVpZ4+67LXHOTOVR0nCE= +github.com/mitchellh/cli v1.1.2/go.mod h1:6iaV0fGdElS6dPBx0EApTxHrcWvmJphyh2n8YBLPPZ4= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc h1:BD7uZqkN8CpjJtN/tScAKiccBikU4dlqe/gNrkRaPY4= +github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc/go.mod h1:HFLT6i9iR4QBOF5rdCyjddC9t59ArqWJV2xx+jwcCMo= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tilinna/clock v1.0.2/go.mod h1:ZsP7BcY7sEEz7ktc0IVy8Us6boDrK8VradlKRUGfOao= +github.com/tilinna/clock v1.1.0 h1:6IQQQCo6KoBxVudv6gwtY8o4eDfhHo8ojA5dP0MfhSs= +github.com/tilinna/clock v1.1.0/go.mod h1:ZsP7BcY7sEEz7ktc0IVy8Us6boDrK8VradlKRUGfOao= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200308013534-11ec41452d41/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= +gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/srv/src/mailinglist/mailer.go b/srv/src/mailinglist/mailer.go new file mode 100644 index 0000000..07d6c3a --- /dev/null +++ b/srv/src/mailinglist/mailer.go @@ -0,0 +1,143 @@ +package mailinglist + +import ( + "context" + "errors" + "strings" + + "github.com/emersion/go-sasl" + "github.com/emersion/go-smtp" + "github.com/mediocregopher/blog.mediocregopher.com/srv/cfg" + "github.com/mediocregopher/mediocre-go-lib/v2/mctx" + "github.com/mediocregopher/mediocre-go-lib/v2/mlog" +) + +// Mailer is used to deliver emails to arbitrary recipients. +type Mailer interface { + Send(to, subject, body string) error +} + +type logMailer struct { + logger *mlog.Logger +} + +// NewLogMailer returns a Mailer instance which will not actually send any +// emails, it will only log to the given Logger when Send is called. +func NewLogMailer(logger *mlog.Logger) Mailer { + return &logMailer{logger: logger} +} + +func (l *logMailer) Send(to, subject, body string) error { + ctx := mctx.Annotate(context.Background(), + "to", to, + "subject", subject, + ) + l.logger.Info(ctx, "would have sent email") + return nil +} + +// NullMailer acts as a Mailer but actually just does nothing. +var NullMailer = nullMailer{} + +type nullMailer struct{} + +func (nullMailer) Send(to, subject, body string) error { + return nil +} + +// MailerParams are used to initialize a new Mailer instance. +type MailerParams struct { + SMTPAddr string + + // Optional, if not given then no auth is attempted. + SMTPAuth sasl.Client + + // The sending email address to use for all emails being sent. + SendAs string +} + +// SetupCfg implement the cfg.Cfger interface. +func (m *MailerParams) SetupCfg(cfg *cfg.Cfg) { + + cfg.StringVar(&m.SMTPAddr, "ml-smtp-addr", "", "Address of SMTP server to use for sending emails for the mailing list") + smtpAuthStr := cfg.String("ml-smtp-auth", "", "user:pass to use when authenticating with the mailing list SMTP server. The given user will also be used as the From address.") + + cfg.OnInit(func(ctx context.Context) error { + if m.SMTPAddr == "" { + return nil + } + + smtpAuthParts := strings.SplitN(*smtpAuthStr, ":", 2) + if len(smtpAuthParts) < 2 { + return errors.New("invalid -ml-smtp-auth") + } + + m.SMTPAuth = sasl.NewPlainClient("", smtpAuthParts[0], smtpAuthParts[1]) + m.SendAs = smtpAuthParts[0] + + return nil + }) +} + +// Annotate implements mctx.Annotator interface. +func (m *MailerParams) Annotate(a mctx.Annotations) { + if m.SMTPAddr == "" { + return + } + + a["smtpAddr"] = m.SMTPAddr + a["smtpSendAs"] = m.SendAs +} + +type mailer struct { + params MailerParams +} + +// NewMailer initializes and returns a Mailer which will use an external SMTP +// server to deliver email. +func NewMailer(params MailerParams) Mailer { + return &mailer{ + params: params, + } +} + +func (m *mailer) Send(to, subject, body string) error { + + msg := []byte("From: " + m.params.SendAs + "\r\n" + + "To: " + to + "\r\n" + + "Subject: " + subject + "\r\n\r\n" + + body + "\r\n") + + c, err := smtp.Dial(m.params.SMTPAddr) + if err != nil { + return err + } + defer c.Close() + + if err = c.Auth(m.params.SMTPAuth); err != nil { + return err + } + + if err = c.Mail(m.params.SendAs, nil); err != nil { + return err + } + + if err = c.Rcpt(to); err != nil { + return err + } + + w, err := c.Data() + if err != nil { + return err + } + + if _, err = w.Write(msg); err != nil { + return err + } + + if err = w.Close(); err != nil { + return err + } + + return c.Quit() +} diff --git a/srv/src/mailinglist/mailinglist.go b/srv/src/mailinglist/mailinglist.go new file mode 100644 index 0000000..fc6e014 --- /dev/null +++ b/srv/src/mailinglist/mailinglist.go @@ -0,0 +1,272 @@ +// Package mailinglist manages the list of subscribed emails and allows emailing +// out to them. +package mailinglist + +import ( + "bytes" + "context" + "errors" + "fmt" + "html/template" + "io" + "net/url" + "strings" + + "github.com/google/uuid" + "github.com/mediocregopher/blog.mediocregopher.com/srv/cfg" + "github.com/mediocregopher/mediocre-go-lib/v2/mctx" + "github.com/tilinna/clock" +) + +var ( + // ErrAlreadyVerified is used when the email is already fully subscribed. + ErrAlreadyVerified = errors.New("email is already subscribed") +) + +// MailingList is able to subscribe, unsubscribe, and iterate through emails. +type MailingList interface { + + // May return ErrAlreadyVerified. + BeginSubscription(email string) error + + // May return ErrNotFound or ErrAlreadyVerified. + FinalizeSubscription(subToken string) error + + // May return ErrNotFound. + Unsubscribe(unsubToken string) error + + // Publish blasts the mailing list with an update about a new blog post. + Publish(postTitle, postURL string) error +} + +// Params are parameters used to initialize a new MailingList. All fields are +// required unless otherwise noted. +type Params struct { + Store Store + Mailer Mailer + Clock clock.Clock + + // PublicURL is the base URL which site visitors can navigate to. + // MailingList will generate links based on this value. + PublicURL *url.URL +} + +// SetupCfg implement the cfg.Cfger interface. +func (p *Params) SetupCfg(cfg *cfg.Cfg) { + publicURLStr := cfg.String("ml-public-url", "http://localhost:4000", "URL this service is accessible at") + + cfg.OnInit(func(ctx context.Context) error { + var err error + if p.PublicURL, err = url.Parse(*publicURLStr); err != nil { + return fmt.Errorf("parsing -public-url: %w", err) + } + + return nil + }) +} + +// Annotate implements mctx.Annotator interface. +func (p *Params) Annotate(a mctx.Annotations) { + a["publicURL"] = p.PublicURL +} + +// New initializes and returns a MailingList instance using the given Params. +func New(params Params) MailingList { + return &mailingList{params: params} +} + +type mailingList struct { + params Params +} + +var beginSubTpl = template.Must(template.New("beginSub").Parse(` +Welcome to the Mediocre Blog mailing list! By subscribing to this mailing list +you are signing up to receive an email everytime a new blog post is published. + +In order to complete your subscription please navigate to the following link: + +{{ .SubLink }} + +This mailing list is built and run using my own hardware and software, and I +solemnly swear that you'll never receive an email from it unless there's a new +blog post. + +If you did not initiate this email, and/or do not wish to subscribe to the +mailing list, then simply delete this email and pretend that nothing ever +happened. + +- Brian +`)) + +func (m *mailingList) BeginSubscription(email string) error { + + emailRecord, err := m.params.Store.Get(email) + + if errors.Is(err, ErrNotFound) { + emailRecord = Email{ + Email: email, + SubToken: uuid.New().String(), + CreatedAt: m.params.Clock.Now(), + } + + if err := m.params.Store.Set(emailRecord); err != nil { + return fmt.Errorf("storing pending email: %w", err) + } + + } else if err != nil { + return fmt.Errorf("finding existing email record: %w", err) + + } else if !emailRecord.VerifiedAt.IsZero() { + return ErrAlreadyVerified + } + + body := new(bytes.Buffer) + err = beginSubTpl.Execute(body, struct { + SubLink string + }{ + SubLink: fmt.Sprintf( + "%s/mailinglist/finalize.html?subToken=%s", + m.params.PublicURL.String(), + emailRecord.SubToken, + ), + }) + + if err != nil { + return fmt.Errorf("executing beginSubTpl: %w", err) + } + + err = m.params.Mailer.Send( + email, + "Mediocre Blog - Please verify your email address", + body.String(), + ) + + if err != nil { + return fmt.Errorf("sending email: %w", err) + } + + return nil +} + +func (m *mailingList) FinalizeSubscription(subToken string) error { + emailRecord, err := m.params.Store.GetBySubToken(subToken) + + if err != nil { + return fmt.Errorf("retrieving email record: %w", err) + + } else if !emailRecord.VerifiedAt.IsZero() { + return ErrAlreadyVerified + } + + emailRecord.VerifiedAt = m.params.Clock.Now() + emailRecord.UnsubToken = uuid.New().String() + + if err := m.params.Store.Set(emailRecord); err != nil { + return fmt.Errorf("storing verified email: %w", err) + } + + return nil +} + +func (m *mailingList) Unsubscribe(unsubToken string) error { + emailRecord, err := m.params.Store.GetByUnsubToken(unsubToken) + + if err != nil { + return fmt.Errorf("retrieving email record: %w", err) + } + + if err := m.params.Store.Delete(emailRecord.Email); err != nil { + return fmt.Errorf("deleting email record: %w", err) + } + + return nil +} + +var publishTpl = template.Must(template.New("publish").Parse(` +A new post has been published to the Mediocre Blog! + +{{ .PostTitle }} +{{ .PostURL }} + +If you're interested then please check it out! + +If you'd like to unsubscribe from this mailing list then visit the following +link instead: + +{{ .UnsubURL }} + +- Brian +`)) + +type multiErr []error + +func (m multiErr) Error() string { + if len(m) == 0 { + panic("multiErr with no members") + } + + b := new(strings.Builder) + fmt.Fprintln(b, "The following errors were encountered:") + for _, err := range m { + fmt.Fprintf(b, "\t- %s\n", err.Error()) + } + + return b.String() +} + +func (m *mailingList) Publish(postTitle, postURL string) error { + + var mErr multiErr + + iter := m.params.Store.GetAll() + for { + emailRecord, err := iter() + if errors.Is(err, io.EOF) { + break + + } else if err != nil { + mErr = append(mErr, fmt.Errorf("iterating through email records: %w", err)) + break + + } else if emailRecord.VerifiedAt.IsZero() { + continue + } + + body := new(bytes.Buffer) + err = publishTpl.Execute(body, struct { + PostTitle string + PostURL string + UnsubURL string + }{ + PostTitle: postTitle, + PostURL: postURL, + UnsubURL: fmt.Sprintf( + "%s/mailinglist/unsubscribe.html?unsubToken=%s", + m.params.PublicURL.String(), + emailRecord.UnsubToken, + ), + }) + + if err != nil { + mErr = append(mErr, fmt.Errorf("rendering publish email template for %q: %w", emailRecord.Email, err)) + continue + } + + err = m.params.Mailer.Send( + emailRecord.Email, + fmt.Sprintf("Mediocre Blog - New Post! - %s", postTitle), + body.String(), + ) + + if err != nil { + mErr = append(mErr, fmt.Errorf("sending email to %q: %w", emailRecord.Email, err)) + continue + } + } + + if len(mErr) > 0 { + return mErr + } + + return nil +} diff --git a/srv/src/mailinglist/store.go b/srv/src/mailinglist/store.go new file mode 100644 index 0000000..f9790c0 --- /dev/null +++ b/srv/src/mailinglist/store.go @@ -0,0 +1,240 @@ +package mailinglist + +import ( + "crypto/sha512" + "database/sql" + "encoding/base64" + "errors" + "fmt" + "io" + "strings" + "time" + + _ "github.com/mattn/go-sqlite3" + migrate "github.com/rubenv/sql-migrate" +) + +var ( + // ErrNotFound is used to indicate an email could not be found in the + // database. + ErrNotFound = errors.New("no record found") +) + +// EmailIterator will iterate through a sequence of emails, returning the next +// email in the sequence on each call, or returning io.EOF. +type EmailIterator func() (Email, error) + +// Email describes all information related to an email which has yet +// to be verified. +type Email struct { + Email string + SubToken string + CreatedAt time.Time + + UnsubToken string + VerifiedAt time.Time +} + +// Store is used for storing MailingList related information. +type Store interface { + + // Set is used to set the information related to an email. + Set(Email) error + + // Get will return the record for the given email, or ErrNotFound. + Get(email string) (Email, error) + + // GetBySubToken will return the record for the given SubToken, or + // ErrNotFound. + GetBySubToken(subToken string) (Email, error) + + // GetByUnsubToken will return the record for the given UnsubToken, or + // ErrNotFound. + GetByUnsubToken(unsubToken string) (Email, error) + + // Delete will delete the record for the given email. + Delete(email string) error + + // GetAll returns all emails for which there is a record. + GetAll() EmailIterator + + Close() error +} + +var migrations = []*migrate.Migration{ + &migrate.Migration{ + Id: "1", + Up: []string{ + `CREATE TABLE emails ( + id TEXT PRIMARY KEY, + email TEXT NOT NULL, + sub_token TEXT NOT NULL, + created_at INTEGER NOT NULL, + + unsub_token TEXT, + verified_at INTEGER + )`, + }, + Down: []string{"DROP TABLE emails"}, + }, +} + +type store struct { + db *sql.DB +} + +// NewStore initializes a new store using the given SQL DB instance. +func NewStore(dbFile string) (Store, error) { + + db, err := sql.Open("sqlite3", dbFile) + if err != nil { + return nil, fmt.Errorf("opening sqlite file: %w", err) + } + + migrations := &migrate.MemoryMigrationSource{Migrations: migrations} + + if _, err := migrate.Exec(db, "sqlite3", migrations, migrate.Up); err != nil { + return nil, fmt.Errorf("running migrations: %w", err) + } + + return &store{ + db: db, + }, nil +} + +func (s *store) emailID(email string) string { + email = strings.ToLower(email) + h := sha512.New() + h.Write([]byte(email)) + return base64.URLEncoding.EncodeToString(h.Sum(nil)) +} + +func (s *store) Set(email Email) error { + _, err := s.db.Exec( + `INSERT INTO emails ( + id, email, sub_token, created_at, unsub_token, verified_at + ) + VALUES + (?, ?, ?, ?, ?, ?) + ON CONFLICT (id) DO UPDATE SET + email=excluded.email, + sub_token=excluded.sub_token, + unsub_token=excluded.unsub_token, + verified_at=excluded.verified_at + `, + s.emailID(email.Email), + email.Email, + email.SubToken, + email.CreatedAt.Unix(), + email.UnsubToken, + sql.NullInt64{ + Int64: email.VerifiedAt.Unix(), + Valid: !email.VerifiedAt.IsZero(), + }, + ) + + return err +} + +var scanCols = []string{ + "email", "sub_token", "created_at", "unsub_token", "verified_at", +} + +type row interface { + Scan(...interface{}) error +} + +func (s *store) scanRow(row row) (Email, error) { + var email Email + var createdAt int64 + var verifiedAt sql.NullInt64 + + err := row.Scan( + &email.Email, + &email.SubToken, + &createdAt, + &email.UnsubToken, + &verifiedAt, + ) + if err != nil { + return Email{}, err + } + + email.CreatedAt = time.Unix(createdAt, 0) + if verifiedAt.Valid { + email.VerifiedAt = time.Unix(verifiedAt.Int64, 0) + } + + return email, nil +} + +func (s *store) scanSingleRow(row *sql.Row) (Email, error) { + email, err := s.scanRow(row) + if errors.Is(err, sql.ErrNoRows) { + return Email{}, ErrNotFound + } + + return email, err +} + +func (s *store) Get(email string) (Email, error) { + row := s.db.QueryRow( + `SELECT `+strings.Join(scanCols, ",")+` + FROM emails + WHERE id=?`, + s.emailID(email), + ) + + return s.scanSingleRow(row) +} + +func (s *store) GetBySubToken(subToken string) (Email, error) { + row := s.db.QueryRow( + `SELECT `+strings.Join(scanCols, ",")+` + FROM emails + WHERE sub_token=?`, + subToken, + ) + + return s.scanSingleRow(row) +} + +func (s *store) GetByUnsubToken(unsubToken string) (Email, error) { + row := s.db.QueryRow( + `SELECT `+strings.Join(scanCols, ",")+` + FROM emails + WHERE unsub_token=?`, + unsubToken, + ) + + return s.scanSingleRow(row) +} + +func (s *store) Delete(email string) error { + _, err := s.db.Exec( + `DELETE FROM emails WHERE id=?`, + s.emailID(email), + ) + return err +} + +func (s *store) GetAll() EmailIterator { + rows, err := s.db.Query( + `SELECT ` + strings.Join(scanCols, ",") + ` + FROM emails`, + ) + + return func() (Email, error) { + if err != nil { + return Email{}, err + + } else if !rows.Next() { + return Email{}, io.EOF + } + return s.scanRow(rows) + } +} + +func (s *store) Close() error { + return s.db.Close() +} diff --git a/srv/src/mailinglist/store_test.go b/srv/src/mailinglist/store_test.go new file mode 100644 index 0000000..25eb150 --- /dev/null +++ b/srv/src/mailinglist/store_test.go @@ -0,0 +1,102 @@ +package mailinglist + +import ( + "io" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestStore(t *testing.T) { + tmpFile, err := ioutil.TempFile(os.TempDir(), "mediocre-blog-mailinglist-store-test-") + if err != nil { + t.Fatal("Cannot create temporary file", err) + } + tmpFilePath := tmpFile.Name() + tmpFile.Close() + + t.Logf("using temporary sqlite file at %q", tmpFilePath) + + t.Cleanup(func() { + if err := os.Remove(tmpFilePath); err != nil { + panic(err) + } + }) + + store, err := NewStore(tmpFilePath) + assert.NoError(t, err) + + t.Cleanup(func() { + assert.NoError(t, store.Close()) + }) + + now := func() time.Time { + return time.Now().Truncate(time.Second) + } + + assertGet := func(t *testing.T, email Email) { + t.Helper() + + gotEmail, err := store.Get(email.Email) + assert.NoError(t, err) + assert.Equal(t, email, gotEmail) + + gotEmail, err = store.GetBySubToken(email.SubToken) + assert.NoError(t, err) + assert.Equal(t, email, gotEmail) + + if email.UnsubToken != "" { + gotEmail, err = store.GetByUnsubToken(email.UnsubToken) + assert.NoError(t, err) + assert.Equal(t, email, gotEmail) + } + } + + assertNotFound := func(t *testing.T, email string) { + t.Helper() + _, err := store.Get(email) + assert.ErrorIs(t, err, ErrNotFound) + } + + // now start actual tests + + // GetAll should not do anything, there's no data + _, err = store.GetAll()() + assert.ErrorIs(t, err, io.EOF) + + emailFoo := Email{ + Email: "foo", + SubToken: "subTokenFoo", + CreatedAt: now(), + } + + // email isn't stored yet, shouldn't exist + assertNotFound(t, emailFoo.Email) + + // Set an email, now it should exist + assert.NoError(t, store.Set(emailFoo)) + assertGet(t, emailFoo) + + // Update the email with an unsub token + emailFoo.UnsubToken = "unsubTokenFoo" + emailFoo.VerifiedAt = now() + assert.NoError(t, store.Set(emailFoo)) + assertGet(t, emailFoo) + + // GetAll should now only return that email + iter := store.GetAll() + gotEmail, err := iter() + assert.NoError(t, err) + assert.Equal(t, emailFoo, gotEmail) + _, err = iter() + assert.ErrorIs(t, err, io.EOF) + + // Delete the email, it should be gone + assert.NoError(t, store.Delete(emailFoo.Email)) + assertNotFound(t, emailFoo.Email) + _, err = store.GetAll()() + assert.ErrorIs(t, err, io.EOF) +} diff --git a/srv/src/pow/pow.go b/srv/src/pow/pow.go new file mode 100644 index 0000000..ada8439 --- /dev/null +++ b/srv/src/pow/pow.go @@ -0,0 +1,321 @@ +// Package pow creates proof-of-work challenges and validates their solutions. +package pow + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/md5" + "crypto/rand" + "crypto/sha512" + "encoding/binary" + "errors" + "fmt" + "hash" + "strconv" + "time" + + "github.com/mediocregopher/blog.mediocregopher.com/srv/cfg" + "github.com/mediocregopher/mediocre-go-lib/v2/mctx" + "github.com/tilinna/clock" +) + +type challengeParams struct { + Target uint32 + ExpiresAt int64 + Random []byte +} + +func (c challengeParams) MarshalBinary() ([]byte, error) { + buf := new(bytes.Buffer) + + var err error + write := func(v interface{}) { + if err != nil { + return + } + err = binary.Write(buf, binary.BigEndian, v) + } + + write(c.Target) + write(c.ExpiresAt) + + if err != nil { + return nil, err + } + + if _, err := buf.Write(c.Random); err != nil { + panic(err) + } + + return buf.Bytes(), nil +} + +func (c *challengeParams) UnmarshalBinary(b []byte) error { + buf := bytes.NewBuffer(b) + + var err error + read := func(into interface{}) { + if err != nil { + return + } + err = binary.Read(buf, binary.BigEndian, into) + } + + read(&c.Target) + read(&c.ExpiresAt) + + if buf.Len() > 0 { + c.Random = buf.Bytes() // whatever is left + } + + return err +} + +// The seed takes the form: +// +// (version)+(signature of challengeParams)+(challengeParams) +// +// Version is currently always 0. +func newSeed(c challengeParams, secret []byte) ([]byte, error) { + buf := new(bytes.Buffer) + buf.WriteByte(0) // version + + cb, err := c.MarshalBinary() + if err != nil { + return nil, err + } + + h := hmac.New(md5.New, secret) + h.Write(cb) + buf.Write(h.Sum(nil)) + + buf.Write(cb) + + return buf.Bytes(), nil +} + +var errMalformedSeed = errors.New("malformed seed") + +func challengeParamsFromSeed(seed, secret []byte) (challengeParams, error) { + h := hmac.New(md5.New, secret) + hSize := h.Size() + + if len(seed) < hSize+1 || seed[0] != 0 { + return challengeParams{}, errMalformedSeed + } + seed = seed[1:] + + sig, cb := seed[:hSize], seed[hSize:] + + // check signature + h.Write(cb) + if !hmac.Equal(sig, h.Sum(nil)) { + return challengeParams{}, errMalformedSeed + } + + var c challengeParams + if err := c.UnmarshalBinary(cb); err != nil { + return challengeParams{}, fmt.Errorf("unmarshaling challenge parameters: %w", err) + } + + return c, nil +} + +// Challenge is a set of fields presented to a client, with which they must +// generate a solution. +// +// Generating a solution is done by: +// +// - Collect up to len(Seed) random bytes. These will be the potential +// solution. +// +// - Calculate the sha512 of the concatenation of Seed and PotentialSolution. +// +// - Parse the first 4 bytes of the sha512 result as a big-endian uint32. +// +// - If the resulting number is _less_ than Target, the solution has been +// found. Otherwise go back to step 1 and try again. +// +type Challenge struct { + Seed []byte + Target uint32 +} + +// Errors which may be produced by a Manager. +var ( + ErrInvalidSolution = errors.New("invalid solution") + ErrExpiredSeed = errors.New("expired seed") +) + +// Manager is used to both produce proof-of-work challenges and check their +// solutions. +type Manager interface { + NewChallenge() Challenge + + // Will produce ErrInvalidSolution if the solution is invalid, or + // ErrExpiredSeed if the seed has expired. + CheckSolution(seed, solution []byte) error +} + +// ManagerParams are used to initialize a new Manager instance. All fields are +// required unless otherwise noted. +type ManagerParams struct { + Clock clock.Clock + Store Store + + // Secret is used to sign each Challenge's Seed, it should _not_ be shared + // with clients. + Secret []byte + + // The Target which Challenges should hit. Lower is more difficult. + // + // Defaults to 0x00FFFFFF + Target uint32 + + // ChallengeTimeout indicates how long before Challenges are considered + // expired and cannot be solved. + // + // Defaults to 1 minute. + ChallengeTimeout time.Duration +} + +func (p *ManagerParams) setDefaults() { + if p.Target == 0 { + p.Target = 0x00FFFFFF + } + if p.ChallengeTimeout == 0 { + p.ChallengeTimeout = 1 * time.Minute + } +} + +// SetupCfg implement the cfg.Cfger interface. +func (p *ManagerParams) SetupCfg(cfg *cfg.Cfg) { + powTargetStr := cfg.String("pow-target", "0x0000FFFF", "Proof-of-work target, lower is more difficult") + powSecretStr := cfg.String("pow-secret", "", "Secret used to sign proof-of-work challenge seeds") + + cfg.OnInit(func(ctx context.Context) error { + p.setDefaults() + + if *powSecretStr == "" { + return errors.New("-pow-secret is required") + } + + powTargetUint, err := strconv.ParseUint(*powTargetStr, 0, 32) + if err != nil { + return fmt.Errorf("parsing -pow-target: %w", err) + } + + p.Target = uint32(powTargetUint) + p.Secret = []byte(*powSecretStr) + + return nil + }) +} + +// Annotate implements mctx.Annotator interface. +func (p *ManagerParams) Annotate(a mctx.Annotations) { + a["powTarget"] = fmt.Sprintf("%x", p.Target) +} + +type manager struct { + params ManagerParams +} + +// NewManager initializes and returns a Manager instance using the given +// parameters. +func NewManager(params ManagerParams) Manager { + params.setDefaults() + return &manager{ + params: params, + } +} + +func (m *manager) NewChallenge() Challenge { + target := m.params.Target + + c := challengeParams{ + Target: target, + ExpiresAt: m.params.Clock.Now().Add(m.params.ChallengeTimeout).Unix(), + Random: make([]byte, 8), + } + + if _, err := rand.Read(c.Random); err != nil { + panic(err) + } + + seed, err := newSeed(c, m.params.Secret) + if err != nil { + panic(err) + } + + return Challenge{ + Seed: seed, + Target: target, + } +} + +// SolutionChecker can be used to check possible Challenge solutions. It will +// cache certain values internally to save on allocations when used in a loop +// (e.g. when generating a solution). +// +// SolutionChecker is not thread-safe. +type SolutionChecker struct { + h hash.Hash // sha512 + sum []byte +} + +// Check returns true if the given bytes are a solution to the given Challenge. +func (s SolutionChecker) Check(challenge Challenge, solution []byte) bool { + if s.h == nil { + s.h = sha512.New() + } + s.h.Reset() + + s.h.Write(challenge.Seed) + s.h.Write(solution) + s.sum = s.h.Sum(s.sum[:0]) + + i := binary.BigEndian.Uint32(s.sum[:4]) + return i < challenge.Target +} + +func (m *manager) CheckSolution(seed, solution []byte) error { + c, err := challengeParamsFromSeed(seed, m.params.Secret) + if err != nil { + return fmt.Errorf("parsing challenge parameters from seed: %w", err) + + } else if now := m.params.Clock.Now().Unix(); c.ExpiresAt <= now { + return ErrExpiredSeed + } + + ok := (SolutionChecker{}).Check( + Challenge{Seed: seed, Target: c.Target}, solution, + ) + + if !ok { + return ErrInvalidSolution + } + + expiresAt := time.Unix(c.ExpiresAt, 0) + if err := m.params.Store.MarkSolved(seed, expiresAt.Add(1*time.Minute)); err != nil { + return fmt.Errorf("marking solution as solved: %w", err) + } + + return nil +} + +// Solve returns a solution for the given Challenge. This may take a while. +func Solve(challenge Challenge) []byte { + + chk := SolutionChecker{} + b := make([]byte, len(challenge.Seed)) + + for { + if _, err := rand.Read(b); err != nil { + panic(err) + } else if chk.Check(challenge, b) { + return b + } + } +} diff --git a/srv/src/pow/pow_test.go b/srv/src/pow/pow_test.go new file mode 100644 index 0000000..cc868b1 --- /dev/null +++ b/srv/src/pow/pow_test.go @@ -0,0 +1,120 @@ +package pow + +import ( + "encoding/hex" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/tilinna/clock" +) + +func TestChallengeParams(t *testing.T) { + tests := []challengeParams{ + {}, + { + Target: 1, + ExpiresAt: 3, + }, + { + Target: 2, + ExpiresAt: -10, + Random: []byte{0, 1, 2}, + }, + { + Random: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + }, + } + + t.Run("marshal_unmarshal", func(t *testing.T) { + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + b, err := test.MarshalBinary() + assert.NoError(t, err) + + var c2 challengeParams + assert.NoError(t, c2.UnmarshalBinary(b)) + assert.Equal(t, test, c2) + + b2, err := c2.MarshalBinary() + assert.NoError(t, err) + assert.Equal(t, b, b2) + }) + } + }) + + secret := []byte("shhh") + + t.Run("to_from_seed", func(t *testing.T) { + + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + seed, err := newSeed(test, secret) + assert.NoError(t, err) + + // generating seed should be deterministic + seed2, err := newSeed(test, secret) + assert.NoError(t, err) + assert.Equal(t, seed, seed2) + + c, err := challengeParamsFromSeed(seed, secret) + assert.NoError(t, err) + assert.Equal(t, test, c) + }) + } + }) + + t.Run("malformed_seed", func(t *testing.T) { + tests := []string{ + "", + "01", + "0000", + "00374a1ad84d6b7a93e68042c1f850cbb100000000000000000000000000000102030405060708A0", // changed one byte from a good seed + } + + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + seed, err := hex.DecodeString(test) + if err != nil { + panic(err) + } + + _, err = challengeParamsFromSeed(seed, secret) + assert.ErrorIs(t, errMalformedSeed, err) + }) + } + }) +} + +func TestManager(t *testing.T) { + clock := clock.NewMock(time.Now().Truncate(time.Hour)) + + store := NewMemoryStore(clock) + defer store.Close() + + mgr := NewManager(ManagerParams{ + Clock: clock, + Store: store, + Secret: []byte("shhhh"), + Target: 0x00FFFFFF, + ChallengeTimeout: 1 * time.Second, + }) + + { + c := mgr.NewChallenge() + solution := Solve(c) + assert.NoError(t, mgr.CheckSolution(c.Seed, solution)) + + // doing again should fail, the seed should already be marked as solved + assert.ErrorIs(t, mgr.CheckSolution(c.Seed, solution), ErrSeedSolved) + } + + { + c := mgr.NewChallenge() + solution := Solve(c) + clock.Add(2 * time.Second) + assert.ErrorIs(t, mgr.CheckSolution(c.Seed, solution), ErrExpiredSeed) + } + +} diff --git a/srv/src/pow/store.go b/srv/src/pow/store.go new file mode 100644 index 0000000..0b5e7d0 --- /dev/null +++ b/srv/src/pow/store.go @@ -0,0 +1,92 @@ +package pow + +import ( + "errors" + "sync" + "time" + + "github.com/tilinna/clock" +) + +// ErrSeedSolved is used to indicate a seed has already been solved. +var ErrSeedSolved = errors.New("seed already solved") + +// Store is used to track information related to proof-of-work challenges and +// solutions. +type Store interface { + + // MarkSolved will return ErrSeedSolved if the seed was already marked. The + // seed will be cleared from the Store once expiresAt is reached. + MarkSolved(seed []byte, expiresAt time.Time) error + + Close() error +} + +type inMemStore struct { + clock clock.Clock + + m map[string]time.Time + l sync.Mutex + closeCh chan struct{} + spinLoopCh chan struct{} // only used by tests +} + +const inMemStoreGCPeriod = 5 * time.Second + +// NewMemoryStore initializes and returns an in-memory Store implementation. +func NewMemoryStore(clock clock.Clock) Store { + s := &inMemStore{ + clock: clock, + m: map[string]time.Time{}, + closeCh: make(chan struct{}), + spinLoopCh: make(chan struct{}, 1), + } + go s.spin(s.clock.NewTicker(inMemStoreGCPeriod)) + return s +} + +func (s *inMemStore) spin(ticker *clock.Ticker) { + defer ticker.Stop() + + for { + select { + case <-ticker.C: + now := s.clock.Now() + + s.l.Lock() + for seed, expiresAt := range s.m { + if !now.Before(expiresAt) { + delete(s.m, seed) + } + } + s.l.Unlock() + + case <-s.closeCh: + return + } + + select { + case s.spinLoopCh <- struct{}{}: + default: + } + } +} + +func (s *inMemStore) MarkSolved(seed []byte, expiresAt time.Time) error { + seedStr := string(seed) + + s.l.Lock() + defer s.l.Unlock() + + if _, ok := s.m[seedStr]; ok { + return ErrSeedSolved + } + + s.m[seedStr] = expiresAt + return nil +} + +func (s *inMemStore) Close() error { + close(s.closeCh) + return nil +} diff --git a/srv/src/pow/store_test.go b/srv/src/pow/store_test.go new file mode 100644 index 0000000..324a40c --- /dev/null +++ b/srv/src/pow/store_test.go @@ -0,0 +1,52 @@ +package pow + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/tilinna/clock" +) + +func TestStore(t *testing.T) { + clock := clock.NewMock(time.Now().Truncate(time.Hour)) + now := clock.Now() + + s := NewMemoryStore(clock) + defer s.Close() + + seed := []byte{0} + + // mark solved should work + err := s.MarkSolved(seed, now.Add(time.Second)) + assert.NoError(t, err) + + // mark again, should not work + err = s.MarkSolved(seed, now.Add(time.Hour)) + assert.ErrorIs(t, err, ErrSeedSolved) + + // marking a different seed should still work + seed2 := []byte{1} + err = s.MarkSolved(seed2, now.Add(inMemStoreGCPeriod*2)) + assert.NoError(t, err) + err = s.MarkSolved(seed2, now.Add(time.Hour)) + assert.ErrorIs(t, err, ErrSeedSolved) + + now = clock.Add(inMemStoreGCPeriod) + <-s.(*inMemStore).spinLoopCh + + // first one should be markable again, second shouldnt + err = s.MarkSolved(seed, now.Add(time.Second)) + assert.NoError(t, err) + err = s.MarkSolved(seed2, now.Add(time.Hour)) + assert.ErrorIs(t, err, ErrSeedSolved) + + now = clock.Add(inMemStoreGCPeriod) + <-s.(*inMemStore).spinLoopCh + + // now both should be expired + err = s.MarkSolved(seed, now.Add(time.Second)) + assert.NoError(t, err) + err = s.MarkSolved(seed2, now.Add(time.Second)) + assert.NoError(t, err) +} |