From c6ba6fdfdee82fad224c98742a03c18207dc24f7 Mon Sep 17 00:00:00 2001 From: Farhan Khan Date: Fri, 24 Jan 2025 21:12:19 +0000 Subject: [PATCH] Updated Dockerfile Accidentally committed a directory twice --- fedilogger/Dockerfile | 11 - fedilogger/config.go | 31 -- fedilogger/config_test.go | 14 - fedilogger/ctl.go | 71 ---- fedilogger/db.go | 20 - fedilogger/fedilogger.go | 118 ------ fedilogger/fedilogger_test.go | 46 --- fedilogger/follow.go | 85 ---- fedilogger/go.mod | 29 -- fedilogger/go.sum | 187 --------- fedilogger/instance.go | 277 ------------- fedilogger/instance_test.go | 60 --- fedilogger/log.go | 58 --- fedilogger/oauth.go | 210 ---------- fedilogger/poll.go | 164 -------- fedilogger/retrieve.go | 287 ------------- fedilogger/retrieve_test.go | 9 - fedilogger/stream.go | 292 -------------- fedilogger/tables.sql | 731 ---------------------------------- fedilogger/testhelper.go | 15 - fedilogger/web.go | 311 --------------- fedilogue/Dockerfile | 12 + restapi/Dockerfile | 3 +- 23 files changed, 14 insertions(+), 3027 deletions(-) delete mode 100644 fedilogger/Dockerfile delete mode 100644 fedilogger/config.go delete mode 100644 fedilogger/config_test.go delete mode 100644 fedilogger/ctl.go delete mode 100644 fedilogger/db.go delete mode 100644 fedilogger/fedilogger.go delete mode 100644 fedilogger/fedilogger_test.go delete mode 100644 fedilogger/follow.go delete mode 100644 fedilogger/go.mod delete mode 100644 fedilogger/go.sum delete mode 100644 fedilogger/instance.go delete mode 100644 fedilogger/instance_test.go delete mode 100644 fedilogger/log.go delete mode 100644 fedilogger/oauth.go delete mode 100644 fedilogger/poll.go delete mode 100644 fedilogger/retrieve.go delete mode 100644 fedilogger/retrieve_test.go delete mode 100644 fedilogger/stream.go delete mode 100644 fedilogger/tables.sql delete mode 100644 fedilogger/testhelper.go delete mode 100644 fedilogger/web.go create mode 100644 fedilogue/Dockerfile diff --git a/fedilogger/Dockerfile b/fedilogger/Dockerfile deleted file mode 100644 index 0e41b39..0000000 --- a/fedilogger/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM golang:latest AS build -WORKDIR /build -COPY go.mod go.sum . -RUN go mod download -COPY . . -RUN go build . - -FROM alpine:latest -WORKDIR /app -COPY --from=build /build/fedilogger /app/fedilogger -ENTRYPOINT ["/app/restapi"] diff --git a/fedilogger/config.go b/fedilogger/config.go deleted file mode 100644 index 9d1a3d5..0000000 --- a/fedilogger/config.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "flag" -) - -// Settings - Configuration file structure -type Settings struct { - Crawl bool - LogLevel int - Hostname string -} - -var settings Settings - -/* Test: TestStringexists */ -func stringexists(needle string, haystack []string) bool { - for _, check := range haystack { - if check == needle { - return true - } - } - return false -} - -func getSettings() { - flag.BoolVar(&settings.Crawl, "c", true, "Crawl mode (default is yes)") - flag.StringVar(&settings.Hostname, "h", "myhostname", "Set your hostname") - flag.IntVar(&settings.LogLevel, "l", 1, "Logging Level:\n 0) No logs\n 1) Reports every 30 seconds\n 2) Errors\n 3) Warnings\n 4) New Connections\n 5) Debugging\n") - flag.Parse() -} diff --git a/fedilogger/config_test.go b/fedilogger/config_test.go deleted file mode 100644 index 40369a8..0000000 --- a/fedilogger/config_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -import ( - "testing" -) - -func TestStringexists(t *testing.T) { - var empty_strings = []string {} - var three_strings = []string {"first", "second", "third"} - - AssertEqual(t, stringexists("amything", empty_strings), false) - AssertEqual(t, stringexists("second", three_strings), true) - AssertEqual(t, stringexists("fourth", three_strings), false) -} diff --git a/fedilogger/ctl.go b/fedilogger/ctl.go deleted file mode 100644 index 6dfd171..0000000 --- a/fedilogger/ctl.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" -) - -func cmdFollow(w http.ResponseWriter, r *http.Request) { - body, err := ioutil.ReadAll(r.Body) - if err != nil { - fmt.Println(err) - return - } - defer r.Body.Close() - - var followtarget interface{} - err = json.Unmarshal(body, &followtarget) - if err != nil { - fmt.Println(err) - return - } - - myMap := followtarget.(map[string]interface{}) - followInbox(myMap["follow"].(string)) - - fmt.Fprintf(w, "{}") -} - -func cmdAdd(w http.ResponseWriter, r *http.Request) { - body, err := ioutil.ReadAll(r.Body) - if err != nil { - fmt.Println(err) - return - } - defer r.Body.Close() - - var addtarget interface{} - err = json.Unmarshal(body, &addtarget) - if err != nil { - fmt.Println(err) - return - } - - myMap := addtarget.(map[string]interface{}) - go StartInstance(myMap["host"].(string)) - - fmt.Fprintf(w, "{}") -} - -func cmdStatus(w http.ResponseWriter, r *http.Request) { - ri_mutex.Lock() - bytes, err := json.Marshal(runninginstances) - if err != nil { - fmt.Println("Error happened", err) - } - ri_mutex.Unlock() - fmt.Fprintf(w, (string(bytes))) -} - -func startctl() { - ctlweb := http.NewServeMux() - ctlweb.HandleFunc("/follow", cmdFollow) - ctlweb.HandleFunc("/status", cmdStatus) - ctlweb.HandleFunc("/add", cmdAdd) - log.Print("Starting HTTP inbox on port 127.0.0.1:5555") - log.Fatal(http.ListenAndServe("127.0.0.1:5555", ctlweb)) - -} diff --git a/fedilogger/db.go b/fedilogger/db.go deleted file mode 100644 index 21f03ec..0000000 --- a/fedilogger/db.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -import ( - "context" - "os" - - "github.com/jackc/pgx/v4/pgxpool" -) - -var pool *pgxpool.Pool - -func getDbPool() *pgxpool.Pool { - // Setup Database - dburl := os.Getenv("DATABASE_URL") - pool, err := pgxpool.Connect(context.Background(), dburl) - if err != nil { - logFatal("Unable to connect to database:", err) - } - return pool -} diff --git a/fedilogger/fedilogger.go b/fedilogger/fedilogger.go deleted file mode 100644 index 83bf9e4..0000000 --- a/fedilogger/fedilogger.go +++ /dev/null @@ -1,118 +0,0 @@ -package main - -import ( - "context" - "net/http" - _ "net/http/pprof" - "regexp" - "runtime" - "sync" - "time" - - "github.com/microcosm-cc/bluemonday" - "git.farhan.codes/farhan/fedilogue/shared" -) - -// Current instances -var runninginstances map[string]shared.RunningInstance -var ri_mutex = &sync.Mutex{} - -func startpprof() { - logInfo("Starting http/pprof on :7777") - logFatal(http.ListenAndServe("127.0.0.1:7777", nil)) -} - -func statusReportHandler() { - for { - StatusReport() - time.Sleep(time.Second * 60) - } -} - -/* Tests: - - TestStatusReport_empty_run - - TestStatusReport_full_content -*/ -func StatusReport() { - running := 0 - keepalive := 0 - unsupported := 0 - - mastodon := 0 - pleroma := 0 - misskey := 0 - other := 0 - ri_mutex.Lock() - for i, o := range runninginstances { - logDebug("Software ", o.Software, " Status: ", o.Status, " instance ", i) - if o.Status == 200 { - running = running + 1 - } else if o.Status == 607 { // Keepalive - keepalive = keepalive + 1 - } else if o.Status == 605 { // Unsupported instance - unsupported = unsupported + 1 - } - - if o.Software == "mastodon" && o.Status == 200 { - mastodon = mastodon + 1 - } else if o.Software == "pleroma" && o.Status == 200 { - pleroma = pleroma + 1 - } else if o.Software == "misskey" && o.Status == 200 { - misskey = misskey + 1 - } else if o.Status == 200 { - other = other + 1 - } - } - ri_mutex.Unlock() - logInfo("Running:", running, " Keepalive:", keepalive, " Unsupported:", unsupported, " Ma:", mastodon, ",P:", pleroma, ",Mi:", misskey, ",O:", other) -} - -func main() { - // Initial Setup - logInit() - runninginstances = make(map[string]shared.RunningInstance) - - getSettings() - go startpprof() - - pool = getDbPool() - - p = bluemonday.NewPolicy() - spaceReg = regexp.MustCompile(`[\s\t\.]+`) - removeHTMLReg = regexp.MustCompile(`<\/?\s*br\s*>`) - re = regexp.MustCompile("^https?://([^/]*)/(.*)$") - matchurl = regexp.MustCompile("http?s://[\\w\\-]+\\.[\\w\\-]+\\S*") - staggeredStartChan = make(chan bool) - - // Start instances located in database - rows, err := pool.Query(context.Background(), "SELECT endpoint FROM instances") - if err != nil { - logErr("Unable to select from instances") - return - } - defer rows.Close() - - go staggeredStart() - go statusReportHandler() - - for rows.Next() { - var endpoint string - err = rows.Scan(&endpoint) - if err != nil { - logErr("Unable to iterate database, exiting.") - return - } - o, exists := GetRunner(endpoint) - if o.Banned == true { - continue // Banned instance - } - if exists == false { - go StartInstance(endpoint) - } - } - - go startctl() - go webmain() - - runtime.Goexit() -} diff --git a/fedilogger/fedilogger_test.go b/fedilogger/fedilogger_test.go deleted file mode 100644 index bf36b81..0000000 --- a/fedilogger/fedilogger_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "strconv" - "testing" - "time" - - "git.farhan.codes/farhan/fedilogue/shared" -) - -func TestStatusReport_empty_run(t *testing.T) { - // Empty Instances run - StatusReport() -} - -func TestStatusReport_full_content(t *testing.T) { - defer func() { - runninginstances = map[string]shared.RunningInstance{} - }() - runninginstances = make(map[string]shared.RunningInstance) - - identifier := 0 - var endpoint string - - test_instance_types := []string{"pleroma", "mastodon", "unknown", ""} - test_statuses := []int{shared.NEW_INSTANCE, shared.RUNNING, shared.UNAUTHORIZED, shared.FORBIDDEN, shared.NOT_FOUND, shared.UNPROCESSABLE_ENTITY, shared.TOOMANYREQUESTS, shared.INTERNAL_ERROR, shared.CLIENT_ISSUE, shared.ONION_PROTOCOL, shared.BAD_RESPONSE, shared.BAD_NODEINFO, shared.UNSUPPORTED_INSTANCE, shared.STREAM_ENDED, shared.KEEPALIVE} - - for _, test_instance_type := range test_instance_types { - for _, test_status := range test_statuses { - a := shared.RunningInstance{} - endpoint = "endpoint" + strconv.Itoa(identifier) + ".test.com" - a.Client = BuildClient(endpoint) - a.Status = test_status - a.Recentactivities = shared.NewUniqueFifo(10) - a.Recentactors = shared.NewUniqueFifo(10) - a.Software = test_instance_type - a.Version = "0." + strconv.Itoa(identifier) - a.LastRun = time.Now().Format(time.RFC3339) - runninginstances[endpoint] = a - identifier = identifier + 1 - } - } - - StatusReport() - -} diff --git a/fedilogger/follow.go b/fedilogger/follow.go deleted file mode 100644 index f3a462a..0000000 --- a/fedilogger/follow.go +++ /dev/null @@ -1,85 +0,0 @@ -package main - -import ( - "bytes" - "crypto/tls" - "crypto/x509" - - "encoding/pem" - "fmt" - "io/ioutil" - "log" - "net/http" - "strings" - "time" - - "github.com/go-fed/httpsig" -) - -func followInbox(inboxurl string) { - matchset := re.FindStringSubmatch(inboxurl) - if matchset == nil { - logWarn("Unable to unregex request", inboxurl) - return - } - inboxhostname := matchset[1] - - keyBytes, err := ioutil.ReadFile("keys/private.pem") - if err != nil { - log.Fatal(err) - } - - pemBlock, _ := pem.Decode(keyBytes) - if pemBlock == nil { - log.Fatal("Invalid PEM format") - } - - privateKey, err := x509.ParsePKCS1PrivateKey(pemBlock.Bytes) - if err != nil { - log.Fatal(err) - } - - jsonRequest := fmt.Sprintf(`{"@context":["https://www.w3.org/ns/activitystreams","https://%[2]s/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://%[2]s/relay","cc":[],"id":"https://%[2]s/activities/bd0614f5-371d-4bd2-88b3-1ee12f7bf42a","object":"%[1]s","state":"pending","to":["%[1]s"],"type":"Follow"}`, inboxurl, settings.Hostname) - - //var jsonBytes []byte - jsonBytes := []byte(jsonRequest) - payload := bytes.NewReader(jsonBytes) - - prefs := []httpsig.Algorithm{httpsig.RSA_SHA256} - digestAlgorithm := httpsig.DigestSha256 - headers := []string{httpsig.RequestTarget, "date", "host"} - signer, _, err := httpsig.NewSigner(prefs, digestAlgorithm, headers, httpsig.Signature, 0) - if err != nil { - log.Fatal(err.Error()) - } - - req, err := http.NewRequest("POST", inboxurl, payload) - if err != nil { - log.Fatal(err.Error()) - } - if payload != nil { - req.Header.Add("content-type", "application/json") - } - req.Header.Add("date", time.Now().UTC().Format(http.TimeFormat)) - req.Header.Add("host", inboxhostname) - - keyID := "https://" + settings.Hostname + "/relay" - - err = signer.SignRequest(privateKey, keyID, req, jsonBytes) - if err != nil { - log.Fatal(err.Error()) - } - req.Header["Signature"][0] = strings.Replace(req.Header["Signature"][0], "algorithm=\"hs2019\"", "algorithm=\"rsa-sha256\"", 1) - - customTransport := http.DefaultTransport.(*http.Transport).Clone() - customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - client := &http.Client{Timeout: 10 * time.Second, Transport: customTransport} - - // client := &http.Client{Timeout: 10 * time.Second} - res, err := client.Do(req) - if err != nil { - log.Fatal(err) - } - - defer res.Body.Close() -} diff --git a/fedilogger/go.mod b/fedilogger/go.mod deleted file mode 100644 index a8c7171..0000000 --- a/fedilogger/go.mod +++ /dev/null @@ -1,29 +0,0 @@ -module git.farhan.codes/farhan/fedilogue/fedilogger - -go 1.23.5 - -require ( - git.farhan.codes/farhan/fedilogue/shared v0.0.0-20250124035748-fa4db6191391 - github.com/go-fed/httpsig v1.1.0 - github.com/google/uuid v1.6.0 - github.com/gorilla/websocket v1.5.3 - github.com/jackc/pgx/v4 v4.18.3 - github.com/microcosm-cc/bluemonday v1.0.27 -) - -require ( - github.com/aymerick/douceur v0.2.0 // indirect - github.com/gorilla/css v1.0.1 // indirect - github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.14.3 // indirect - github.com/jackc/pgio v1.0.0 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.3.3 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgtype v1.14.0 // indirect - github.com/jackc/puddle v1.3.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect -) diff --git a/fedilogger/go.sum b/fedilogger/go.sum deleted file mode 100644 index 4f10f5d..0000000 --- a/fedilogger/go.sum +++ /dev/null @@ -1,187 +0,0 @@ -git.farhan.codes/farhan/fedilogue/shared v0.0.0-20250124035748-fa4db6191391 h1:/pt4xgnZGxS/pC+CpRQ498gfbOy1E3o16eGKch4AR10= -git.farhan.codes/farhan/fedilogue/shared v0.0.0-20250124035748-fa4db6191391/go.mod h1:KT2OwXD90rPF8qPL/pJVg4WRwq4qvLQcMnoyldckXXw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= -github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= -github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= -github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= -github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= -github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= -github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= -github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= -github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= -github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= -github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/fedilogger/instance.go b/fedilogger/instance.go deleted file mode 100644 index d7eb6c8..0000000 --- a/fedilogger/instance.go +++ /dev/null @@ -1,277 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "net" - "net/http" - "time" - - "git.farhan.codes/farhan/fedilogue/shared" -) - -var staggeredStartChan chan bool - -func DoTries(o *shared.RunningInstance, req *http.Request) (*http.Response, error) { - var resp *http.Response - var err error - - for tries := 0; tries < 10; tries++ { - resp, err = o.Client.Do(req) - if err != nil { - // URL.Scheme, Host, Path Opaque - logWarn("Failure connecting to "+req.URL.Scheme+"://"+req.URL.Host+req.URL.Path+", attempt ", tries+1, ", sleeping for 5 minutes: ", err) - time.Sleep(time.Minute * 5) - continue - } - break - } - return resp, err -} - -func BuildClient(endpoint string) http.Client { - logDebug("BuildClient for ", endpoint) - // Test: TestBuildClient, TestBuildClientProxy - /* The seemingly unused 'endpoint' variable is for proxying based on endpoint, ie for Tor */ - tr := &http.Transport{ - MaxIdleConns: 2, - IdleConnTimeout: 3600 * time.Second, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - } - client := http.Client{Transport: tr} - - return client -} - -func GetRunner(endpoint string) (shared.RunningInstance, bool) { - // Tests: TestGetRunnerNonExist, TestGetRunnerExists - ri_mutex.Lock() - o, exists := runninginstances[endpoint] - - if exists == false { - o = shared.RunningInstance{} - selectRet := pool.QueryRow(context.Background(), "SELECT banned, alwaysbot FROM instances WHERE endpoint = $1", endpoint) - err := selectRet.Scan(&o.Banned, &o.Alwaysbot) - if err != nil { - logDebug("Did not find instance in database: ", endpoint) - } - if o.Banned == true { - logInfo("Banned instance: ", endpoint) - } else { - logDebug("Building runner for: ", endpoint) - o.Client = BuildClient(endpoint) - o.Status = shared.KEEPALIVE - o.Recentactivities = shared.NewUniqueFifo(10) - o.Recentactors = shared.NewUniqueFifo(10) - } - runninginstances[endpoint] = o - } - ri_mutex.Unlock() - - return o, exists -} - -func UpdateRunner(endpoint string, o shared.RunningInstance) { - // Tests: None necessary - ri_mutex.Lock() - runninginstances[endpoint] = o - ri_mutex.Unlock() -} - -func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningInstance { - /* Checking order - * Mastodon/Pleroma/Misskey - * Um..nothing else yet - */ - logDebug("GetInstanceInfo for ", endpoint) - var nodeinfo shared.NodeInfo - pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json" - - // Checking Mastodon and Pleroma (with .json) - reqjson, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil) - reqjson.Header.Set("User-Agent", "Tusky") - - pleromastodon_api_resp, err := DoTries(&o, reqjson) - if err != nil { - o.Software = "Unsupported" - return o - } else { - defer pleromastodon_api_resp.Body.Close() - } - - if pleromastodon_api_resp.StatusCode == 200 { - err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo) - if err == nil { - o.Software = nodeinfo.Software.Name - o.Version = nodeinfo.Software.Version - o.LastRun = time.Now().Format(time.RFC3339) - defer pleromastodon_api_resp.Body.Close() - return o - } - } - - // Checking for Misskey (without .json) - misskey_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0" - req, _ := http.NewRequest("GET", misskey_nodeinfo_uri, nil) - req.Header.Set("User-Agent", "Tusky") - - misskey_api_resp, err := DoTries(&o, req) - if err != nil { - o.Software = "Unsupported" - return o - } else { - defer misskey_api_resp.Body.Close() - } - - if misskey_api_resp.StatusCode == 200 { - err = json.NewDecoder(misskey_api_resp.Body).Decode(&nodeinfo) - if err == nil { - o.Software = nodeinfo.Software.Name - o.Version = nodeinfo.Software.Version - o.LastRun = time.Now().Format(time.RFC3339) - defer misskey_api_resp.Body.Close() - return o - } - } - - // Check for Lemmy (With Json) - lemmy_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.1" - req, _ = http.NewRequest("GET", lemmy_nodeinfo_uri, nil) - req.Header.Set("User-Agent", "Tusky") - - lemmy_api_resp, err := DoTries(&o, req) - if err != nil { - o.Software = "Unsupported" - return o - } else { - defer lemmy_api_resp.Body.Close() - } - - if lemmy_api_resp.StatusCode == 200 { - err = json.NewDecoder(lemmy_api_resp.Body).Decode(&nodeinfo) - if err == nil { - logDebug("Found a new Lemmy instance: " + endpoint) - o.Software = nodeinfo.Software.Name - o.Version = nodeinfo.Software.Version - o.LastRun = time.Now().Format(time.RFC3339) - defer lemmy_api_resp.Body.Close() - return o - } - } - - // Unsupported Software - o.Software = "Unsupported" - o.Version = "Unknown" - - return o -} - -func LogInstance(endpoint string, o shared.RunningInstance) bool { - logDebug("LogInstance: ", endpoint) - selectRet := pool.QueryRow(context.Background(), "SELECT FROM instances WHERE endpoint = $1", endpoint) - err := selectRet.Scan() - if err == nil { - return true // Endpoint already in database, continuing - } - - _, err = pool.Exec(context.Background(), "INSERT INTO instances (endpoint, state, software) VALUES($1, $2, $3)", endpoint, "", o.Software) - if err != nil { - logWarn("Error inserting ", endpoint+" into `instances`: ", err) - return true - } - - return false -} - -func CheckInstance(newinstance string, callerEndpoint string) { - logDebug("checkInstance: ", newinstance) - if settings.Crawl == true { - // Skip over this if its the same as the endpoint or empty - if newinstance == callerEndpoint || newinstance == "" { - return - } - - var err error - for attempt := 0; attempt > 5; attempt = attempt + 1 { - _, err = net.LookupHost(newinstance) - if err != nil { - logDebug("Unable to resolve "+newinstance+" attempt ", attempt, "/5. Sleeping for 30 seconds") - time.Sleep(time.Second * 30) - continue - } - break - } - if err != nil { - logWarn("Unable to resolve ", newinstance, " after 5 attempts, giving up: ", err) - return - } - - // Skip over this if its the same as the endpoint - if newinstance == callerEndpoint { - return - } - - // Going forward, this might be merged into GetRunner - ri_mutex.Lock() - o, exists := runninginstances[newinstance] - if exists == false || o.Status == shared.KEEPALIVE { - m := shared.RunningInstance{} - m.Client = BuildClient(newinstance) - m.Recentactivities = shared.NewUniqueFifo(10) - m.Recentactors = shared.NewUniqueFifo(10) - runninginstances[newinstance] = m - go StartInstance(newinstance) - } - ri_mutex.Unlock() - } -} - -func staggeredStart() { - for { - _: - <-staggeredStartChan - time.Sleep(500 * time.Millisecond) - } -} - -func StartInstance(endpoint string) { - staggeredStartChan <- true - logInfo("Starting " + endpoint) - - // Check if exists. If so, get the object. If not, create it - o, _ := GetRunner(endpoint) - if o.Banned == true { - logInfo("Ignoring banned instance: ", endpoint) - return // banned instance - } - - o = GetInstanceInfo(endpoint, o) - UpdateRunner(endpoint, o) - LogInstance(endpoint, o) - - if o.Software == "pleroma" { - logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version) - o.CaptureType = "Stream" - UpdateRunner(endpoint, o) - // PollMastodonPleroma(endpoint, &o) - StreamPleroma(endpoint) - } else if o.Software == "mastodon" { - logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version) - o.CaptureType = "Stream" - UpdateRunner(endpoint, o) - StreamMastodon(endpoint, &o) - } else if o.Software == "misskey" { - logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version) - o.CaptureType = "Stream" - UpdateRunner(endpoint, o) - StreamMisskey(endpoint) - } else { - o.Status = 605 - UpdateRunner(endpoint, o) - logConn("Unsupported endpoint " + endpoint) - } -} diff --git a/fedilogger/instance_test.go b/fedilogger/instance_test.go deleted file mode 100644 index 9209d27..0000000 --- a/fedilogger/instance_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package main - -import ( - "net" - "net/http" - "reflect" - "testing" - "time" - - "git.farhan.codes/farhan/fedilogue/shared" -) - -func TestBuildClient(t *testing.T) { - tr := &http.Transport{ - MaxIdleConns: 2, - IdleConnTimeout: 3600 * time.Second, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - } - want := http.Client{Transport: tr} - have := BuildClient("testdomain.com") - - if reflect.DeepEqual(want, have) { - t.Fatalf("TestBuildClient client different from expected.") - } -} - -func TestBuildClientProxy(t *testing.T) { - // Currently not implemented -} - -func TestGetRunnerNonExist(t *testing.T) { - // Currently not implemented -} - -func TestGetRunnerExists(t *testing.T) { - defer func() { - runninginstances = map[string]shared.RunningInstance{} - }() - - want_o := shared.RunningInstance{} - want_o.Client = BuildClient("some-non-existent-domain.tld") - want_o.Status = shared.KEEPALIVE - want_o.Recentactivities = shared.NewUniqueFifo(10) - want_o.Recentactors = shared.NewUniqueFifo(10) - runninginstances["some-non-existent-domain.tld"] = want_o - - want_exists := true - _, have_exists := GetRunner("some-non-existent-domain.tld") - - if have_exists != want_exists { - t.Fatalf("TestGetRunnerBlank expected %v, got %v", want_exists, have_exists) - } - // if reflect.DeepEqual(want_o, have_o) { - // t.Fatalf("TestGetRunnerExists failed, should have the same value") - // } -} diff --git a/fedilogger/log.go b/fedilogger/log.go deleted file mode 100644 index 19eac15..0000000 --- a/fedilogger/log.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "log" - "os" -) - -var ( - _logInfo *log.Logger - _logErr *log.Logger - _logWarn *log.Logger - _logConn *log.Logger - _logDebug *log.Logger - _logFatal *log.Logger -) - -func logInit() { - _logInfo = log.New(os.Stdout, "INFO: ", log.Lmsgprefix|log.Ldate|log.Ltime) - _logErr = log.New(os.Stdout, "ERR: ", log.Lmsgprefix|log.Ldate|log.Ltime) - _logWarn = log.New(os.Stdout, "WARN: ", log.Lmsgprefix|log.Ldate|log.Ltime) - _logConn = log.New(os.Stdout, "CONN: ", log.Lmsgprefix|log.Ldate|log.Ltime) - _logDebug = log.New(os.Stdout, "DEBUG: ", log.Lmsgprefix|log.Ldate|log.Ltime) - _logFatal = log.New(os.Stdout, "FATAL: ", log.Lmsgprefix|log.Ldate|log.Ltime) -} - -func logInfo(s ...interface{}) { - if settings.LogLevel >= 1 { - _logInfo.Print(s...) - } -} - -func logErr(s ...interface{}) { - if settings.LogLevel >= 2 { - _logErr.Print(s...) - } -} - -func logWarn(s ...interface{}) { - if settings.LogLevel >= 3 { - _logWarn.Print(s...) - } -} - -func logConn(s ...interface{}) { - if settings.LogLevel >= 4 { - _logConn.Print(s...) - } -} - -func logDebug(s ...interface{}) { - if settings.LogLevel >= 5 { - _logDebug.Print(s...) - } -} - -func logFatal(s ...interface{}) { - _logFatal.Fatal(s...) -} diff --git a/fedilogger/oauth.go b/fedilogger/oauth.go deleted file mode 100644 index 641d52e..0000000 --- a/fedilogger/oauth.go +++ /dev/null @@ -1,210 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "encoding/json" - "io" - "io/ioutil" - "net/http" - "os" - - "git.farhan.codes/farhan/fedilogue/shared" -) - -type OAuth struct { - Access_token string `"json:access_token"` - Created_at int `"json:created_at"` - Expires_in int64 `"json:Expires_in"` - Refresh_token string `"json:refresh_token"` -} - -type authError struct { - msg string -} - -func (e *authError) Error() string { - return e.msg -} - -func register_client(endpoint string, o *shared.RunningInstance) error { - requestBodymap, _ := json.Marshal(map[string]string{ - "client_name": "Tusky", // Hard-coded in for now... - "scopes": "read write follow push", - "redirect_uris": "urn:ietf:wg:oauth:2.0:oob", - }) - requestBodybytes := bytes.NewBuffer(requestBodymap) - - api_base_apps := "https://" + endpoint + "/api/v1/apps" - - resp, err := o.Client.Post(api_base_apps, "application/json", requestBodybytes) - if err != nil { - logErr("Unable to connect to "+api_base_apps+" ", err) - return err - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - logErr("Unable to read HTTP response: ", err) - o.Client_id = "" - o.Client_secret = "" - return err - } - - bodymap := make(map[string]string) - err = json.Unmarshal(body, &bodymap) - if err != nil { - logErr("Unable to parse response from "+endpoint+": ", err) - o.Client_id = "" - o.Client_secret = "" - return err - } - - client_file := "clients/" + endpoint - - f, err := os.Create("clients/" + endpoint) - if err != nil { - logErr("Unable to create "+client_file+": ", err) - o.Client_id = "" - o.Client_secret = "" - return err - } - defer f.Close() - - _, err = io.WriteString(f, bodymap["client_id"]+"\n") - if err != nil { - logErr("Unable to write client_id line to file "+client_file+": ", err) - o.Client_id = bodymap["client_id"] - o.Client_secret = bodymap["client_secret"] - return nil - } - _, err = io.WriteString(f, bodymap["client_secret"]+"\n") - if err != nil { - logErr("Unable to write client_secret to file "+client_file+": ", err) - o.Client_id = bodymap["client_id"] - o.Client_secret = bodymap["client_secret"] - return nil - } - - o.Client_id = bodymap["client_id"] - o.Client_secret = bodymap["client_secret"] - return nil -} - -func get_client(endpoint string, o *shared.RunningInstance) error { - var err error - client_file := "clients/" + endpoint - _, err = os.Stat(client_file) - if os.IsNotExist(err) == false { // The file exists - f, err := os.Open(client_file) - if err != nil { - logErr("Unable to open " + client_file + ", creating new client") - return err - } - defer f.Close() - - rd := bufio.NewReader(f) - - client_id_bin, _, err := rd.ReadLine() - o.Client_id = string(client_id_bin) - if err != nil { - logErr("Unable to read client_id line of " + client_file + ", building new client") - return err - } - client_secret_bin, _, err := rd.ReadLine() - o.Client_secret = string(client_secret_bin) - if err != nil { - logErr("Unable to read client_secret line of " + client_file + ", building new client") - return err - } - - return nil - } else { - return register_client(endpoint, o) - } - - return nil -} - -func oauth_login(endpoint string, o *shared.RunningInstance, username string, password string) (OAuth, error) { - authMap, err := json.Marshal(map[string]string{ - "username": username, - "password": password, - "redirect_uri": "urn:ietf:wg:oauth:2.0:oob", - "grant_type": "password", - "client_name": "Tusky", - "scope": "read write follow push", - "client_id": o.Client_id, - "client_secret": o.Client_secret, - }) - - if err != nil { - logErr("Unable to create Authentication map for " + endpoint) - return OAuth{}, err - } - - authMapbytes := bytes.NewBuffer(authMap) - - resp, err := http.Post("https://"+endpoint+"/oauth/token", "application/json", authMapbytes) - if err != nil { - logErr("Cannot connect to "+endpoint+": ", err) - return OAuth{}, err - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - logErr("Unable to read response data for "+endpoint+": ", err) - return OAuth{}, err - } - - if resp.StatusCode == 400 { - logErr("Unable to authenticate to " + endpoint) - return OAuth{}, &authError{"Authentication error"} - } - - oauthData := OAuth{} - err = json.Unmarshal(body, &oauthData) - if err != nil { - logErr("Unable to parse json data for "+endpoint+": ", err) - return OAuth{}, err - } - - return oauthData, nil -} - -func oauth_refresh(endpoint string, client_id string, client_secret string, refresh_token string) (OAuth, error) { - authMap, _ := json.Marshal(map[string]string{ - "redirect_uri": "urn:ietf:wg:oauth:2.0:oob", - "grant_type": "refresh_token", - "scope": "read write follow push", - "refresh_token": refresh_token, - "client_id": client_id, - "client_secret": client_secret, - }) - - authMapbytes := bytes.NewBuffer(authMap) - - resp, err := http.Post("https://"+endpoint+"/oauth/token", "application/json", authMapbytes) - if err != nil { - logErr("Unable to connect to "+endpoint+": ", err) - return OAuth{}, err - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - logErr("Unable to read response data for "+endpoint+": ", err) - return OAuth{}, err - } - - oauthData := OAuth{} - err = json.Unmarshal(body, &oauthData) - if err != nil { - logErr("Unable to parse json data for "+endpoint+": ", err) - return oauthData, err - } - - return oauthData, nil -} diff --git a/fedilogger/poll.go b/fedilogger/poll.go deleted file mode 100644 index b7a9c8b..0000000 --- a/fedilogger/poll.go +++ /dev/null @@ -1,164 +0,0 @@ -package main - -import ( - "encoding/json" - "io/ioutil" - "net/http" - "time" - - "git.farhan.codes/farhan/fedilogue/shared" -) - -type ImageData struct { - Type string `"json:type"` - Url string `"json:url"` -} - -type PublicKeyData struct { - Id string `"json:id"` - Owner string `"json:owner"` - PublicKeyPem string `"json:publicKeyPem"` -} - -type UserInfo struct { - Id string `"json:id"` - Type string `"json:type"` - Following string `"json:following"` - Followers string `"json:followers"` - Inbox string `"json:inbox"` - Outbox string `"json:outbox"` - Featured string `"json:featured"` - PreferredUsername string `"json:preferredUsername"` - PublicKey PublicKeyData `"json:publicKeyPem"` - Name string `"json:name"` - Summary string `"json:summary"` - Url string `"json:Url"` - Icon ImageData `"json:icon"` - Image ImageData `"json:image"` -} - -type PostInfo struct { - Id string `"json:id"` - Type string `"json:type"` - Published string `"json:published"` - Url string `"json:Url"` - Content string `"json:content"` -} - -func PollMastodonPleroma(endpoint string, o *shared.RunningInstance) { - newactivities := make([]shared.ReportActivity, 0) - - min_id := "" - - parsing_error := 0 - use_auth := false - - var last_refresh int64 - var client_id string - var client_secret string - var oauthData OAuth - - for { - ri_mutex.Lock() - m := runninginstances[endpoint] - ri_mutex.Unlock() - - api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&since_id=" + min_id - req, err := http.NewRequest("GET", api_timeline, nil) - req.Header.Set("User-Agent", "Tusky") - if err != nil { - logFatal("Unable to create new request for "+endpoint+": ", err) - return - } - - if use_auth == true { - if time.Now().Unix() > last_refresh+oauthData.Expires_in { - oauthData, err = oauth_refresh(endpoint, client_id, client_secret, oauthData.Refresh_token) - if err != nil { - logWarn("Unable to refresh oauth token for "+endpoint+": ", err) - return - } - last_refresh = time.Now().Unix() - } - req.Header.Add("Authorization", oauthData.Access_token) - } - - m.LastRun = time.Now().Format(time.RFC3339) - resp, err := DoTries(o, req) - if err != nil { - m.Status = shared.CLIENT_ISSUE - ri_mutex.Lock() - runninginstances[endpoint] = m - ri_mutex.Unlock() - logWarn("Giving up on "+endpoint+": ", err.Error()) - return - } - - if resp.StatusCode == shared.TOOMANYREQUESTS { // Short Delay, 30 seconds - logWarn("Delaying "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay") - _, _ = ioutil.ReadAll(resp.Body) - resp.Body.Close() // Release as soon as done - m.Status = resp.StatusCode - ri_mutex.Lock() - runninginstances[endpoint] = m - ri_mutex.Unlock() - time.Sleep(time.Second * 30) - continue - } else if resp.StatusCode == shared.INTERNAL_ERROR { // Longer delay, 1 hour - logWarn("Suspending "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay") - _, _ = ioutil.ReadAll(resp.Body) - resp.Body.Close() // Release as soon as done - m.Status = 765 - ri_mutex.Lock() - runninginstances[endpoint] = m - ri_mutex.Unlock() - time.Sleep(time.Second * 3600) - continue - } else if resp.StatusCode != 200 { // Crash - logErr("Terminating "+endpoint+", gave status ", resp.StatusCode) - _, _ = ioutil.ReadAll(resp.Body) - resp.Body.Close() // Release as soon as done - m.Status = resp.StatusCode - ri_mutex.Lock() - runninginstances[endpoint] = m - ri_mutex.Unlock() - return - } - - err = json.NewDecoder(resp.Body).Decode(&newactivities) - resp.Body.Close() // Release as soon as done - if err != nil { - if parsing_error > 5 { - m.Status = shared.BAD_RESPONSE - ri_mutex.Lock() - runninginstances[endpoint] = m - ri_mutex.Unlock() - logErr("Giving up on " + endpoint + " after 5 unmarshal errors.") - return - } - parsing_error = parsing_error + 1 - time.Sleep(time.Second * 30) - } - - m.Status = shared.RUNNING - ri_mutex.Lock() - runninginstances[endpoint] = m - ri_mutex.Unlock() - - for _, newactivity := range newactivities { - go check_activity(newactivity.Uri) - matchset := re.FindStringSubmatch(newactivity.Uri) - if matchset != nil { - newinstance := matchset[1] - - // Check min_id - if newactivity.Id > min_id { - min_id = newactivity.Id - } - - go CheckInstance(newinstance, endpoint) - } - } - time.Sleep(time.Second * 10) - } -} diff --git a/fedilogger/retrieve.go b/fedilogger/retrieve.go deleted file mode 100644 index 5059414..0000000 --- a/fedilogger/retrieve.go +++ /dev/null @@ -1,287 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "html" - "io/ioutil" - "net/http" - "regexp" - "strings" - "time" - - "github.com/microcosm-cc/bluemonday" -) - -var p *bluemonday.Policy -var spaceReg *regexp.Regexp -var removeHTMLReg *regexp.Regexp -var re *regexp.Regexp -var matchurl *regexp.Regexp - -type ImageType struct { - Url string `json:"url"` -} - -type PublicKeyType struct { - PublicKeyPem string `json:"publicKeyPem"` -} - -type ActorJson struct { - id int - Uri string `json:"id"` - Type string `json:"type"` - Inbox string `json:"inbox"` - Outbox string `json:"outbox"` - Followers string `json:"followers"` - Following string `json:"following"` - Url string `json:"url"` - PreferredUsername string `json:"preferredUsername"` - Name string `json:"name"` - Summary string `json:"summary"` - Icon ImageType `json:"icon"` - Image ImageType `json:"image"` - PublicKey PublicKeyType `json:"publicKey"` - - bot bool - instance string -} - -type TagType struct { - Type string `json:"type"` - Name string `json:"name"` -} - -type PostJson struct { - id int - Uri string `json:"id"` - InReplyTo string `json:"inReplyTo"` - - normalized string - receivedAt time.Time `json:"created_at"` - - Content string `json:"content"` - Conversation string `json:"conversation"` - Published time.Time `json:"published"` - Summary string `json:"summary"` - Tag []TagType `json:"tag"` - To []string `json:"to"` - Type string `json:"type"` - - Actor string `json:"actor"` - AttributedTo string `json:"attributedTo"` - - bot bool - instance string -} - -func check_activity(uri string) { - logDebug("Retrieving: " + uri) - var activityjson PostJson - - // Ignore invalid URIs - endslash := strings.Index(uri[8:], "/") - if endslash == -1 { - return - } - activityjson.instance = uri[8 : endslash+8] - - o, _ := GetRunner(activityjson.instance) - if o.Banned == true { - logDebug("Ignoring banned instance: ", uri) - return // Banned instance - } - - // Check if there were any recent requests on this - o.Recentactivities.Mu.Lock() - i, _ := o.Recentactivities.Contains(uri) - if i != -1 { - logDebug("Ignoring cached recent request: ", uri) - o.Recentactivities.Mu.Unlock() - return - } - - o.Recentactivities.Add(uri, "") // Added blank entry - o.Recentactivities.Mu.Unlock() - var jsondocument string - - selectRet := pool.QueryRow(context.Background(), "SELECT FROM activities WHERE document->>'id' = $1", uri) - err := selectRet.Scan() - if err == nil { - logDebug("Already in database, ignoring: ", uri) - return - } - - req, _ := http.NewRequest("GET", uri, nil) - req.Header.Set("User-Agent", "Tusky") - req.Header.Add("Accept", "application/ld+json") - - resp, err := DoTries(&o, req) - if err != nil { - logDebug("Gave up after multiple tries: ", uri) - return - } - - if resp.StatusCode != 200 { - logDebug("Non-200 response code for ", uri, " was ", resp.StatusCode) - resp.Body.Close() - return - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - logDebug("Failed to read the reply: ", uri) - return - } - resp.Body.Close() - jsondocument = string(body) - err = json.Unmarshal(body, &activityjson) - if err != nil { - logDebug("Failed to Unmarshal, err: ", err, " uri: ", uri) - return - } - - if activityjson.InReplyTo != "" && activityjson.InReplyTo != uri { - if activityjson.InReplyTo != uri { - go check_activity(activityjson.InReplyTo) - } - } - - // If AttributedTo is blank, this is likely an authentication failure - // For now, skip it... - if activityjson.AttributedTo == "" { - logDebug("AttributedTo field is blank, dropping for ", uri) - return - } - - // This must be done BEFORE the `INSERT INTO activities'` below - actorjson := check_actor(activityjson.AttributedTo) - if actorjson == nil { - logDebug("Failed to add actor, dropping post: ", uri) - return - } - if actorjson.bot || o.Alwaysbot { - activityjson.bot = true - } - - activityjson.normalized = removeHTMLReg.ReplaceAllString(activityjson.Content, " ") - activityjson.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(activityjson.normalized))) - activityjson.normalized = matchurl.ReplaceAllString(activityjson.normalized, "") - activityjson.normalized = spaceReg.ReplaceAllString(activityjson.normalized, " ") - - var hashtags []string - for _, tag := range activityjson.Tag { - if tag.Type == "Hashtag" { - hashtags = append(hashtags, strings.ToLower(tag.Name)) - } - } - _, err = pool.Exec(context.Background(), "INSERT INTO activities (document, normalized, instance, hashtags, bot) VALUES($1, $2, $3, $4, $5)", jsondocument, activityjson.normalized, activityjson.instance, hashtags, activityjson.bot) - if err != nil { - logWarn("Error inserting ", uri, " into `activities`: ", err) - return - } - - for _, to := range activityjson.To { - if to != "https://www.w3.org/ns/activitystreams#Public" && to != "" { - if strings.HasSuffix(to, "/followers") { - // This check is very much a bad solution, may consider removing the entire for-loop - continue - } - go check_actor(to) - } - } - -} - -/* Test: TestCheck_actor */ -func check_actor(uri string) *ActorJson { - actorjson := &ActorJson{} - - if len(uri) <= 7 { - return nil // Bad actor - } - - endslash := strings.Index(uri[8:], "/") - if endslash == -1 { - return nil // Bad actor - } - actorjson.instance = uri[8 : endslash+8] - - // Check if there were any recent requests on this - o, _ := GetRunner(actorjson.instance) - if o.Banned { - logDebug("Banned actor: ", uri) - return nil // Banned actor - } - o.Recentactors.Mu.Lock() - i, cachedactorjson := o.Recentactors.Contains(uri) - if i != -1 { - o.Recentactors.Mu.Unlock() - cachedactorjson := cachedactorjson.(*ActorJson) - return cachedactorjson - } - o.Recentactors.Mu.Unlock() - - selectRet := pool.QueryRow(context.Background(), "SELECT document FROM actors WHERE document->>'id' = $1", uri) - err := selectRet.Scan(&actorjson) - if err == nil { - return actorjson // Actor already in database, good! - } - - req, _ := http.NewRequest("GET", uri, nil) - req.Header.Set("User-Agent", "Tusky") - req.Header.Add("Accept", "application/ld+json") - - var resp *http.Response - tries := 0 - for { - resp, err = o.Client.Do(req) - if err != nil { - if tries > 10 { - logErr("Unable to connect to " + uri + " attempt 10/10, giving up.") - return nil // Unable to connect to host after 10 attempts - } - logWarn("Unable to connect to "+uri+", attempt ", tries+1, "+/10 sleeping for 30 seconds.") - time.Sleep(time.Second * 30) - tries = tries + 1 - continue - } - break - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - logWarn("Unable to read body from ", uri) - return nil // Unable to read body of message - } - resp.Body.Close() - - jsondocument := string(body) - - err = json.Unmarshal(body, &actorjson) - if err != nil { - logWarn("Unable to unmarshal body from ", uri) - return nil // Unable to unmarshal body of message - } - o.Recentactors.Mu.Lock() - o.Recentactors.Add(uri, actorjson) - o.Recentactors.Mu.Unlock() - - var bot bool - if actorjson.Type == "Service" { - actorjson.bot = true - } else { - actorjson.bot = o.Alwaysbot // default on host's classification - } - - _, err = pool.Exec(context.Background(), "INSERT INTO actors (document, instance, bot) VALUES($1, $2, $3)", jsondocument, actorjson.instance, bot) - if err != nil { - logWarn("Error inserting ", uri, " into `actors`: ", err) - return nil // Unable to insert actor - } - - o.Recentactors.Mu.Lock() - o.Recentactors.Add(uri, actorjson) - o.Recentactors.Mu.Unlock() - return actorjson // Successful -} diff --git a/fedilogger/retrieve_test.go b/fedilogger/retrieve_test.go deleted file mode 100644 index 2681715..0000000 --- a/fedilogger/retrieve_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "testing" -) - -func TestCheck_actor(t *testing.T) { - // Currently not implemented -} diff --git a/fedilogger/stream.go b/fedilogger/stream.go deleted file mode 100644 index 2096070..0000000 --- a/fedilogger/stream.go +++ /dev/null @@ -1,292 +0,0 @@ -package main - -import ( - "bufio" - "encoding/json" - - "net/http" - "net/url" - "strings" - "time" - - "github.com/google/uuid" - "github.com/gorilla/websocket" - "git.farhan.codes/farhan/fedilogue/shared" -) - -type MisskeyReply struct { - Type string `json:"type"` - Body MisskeyReplyBody `json:"body"` -} - -type MisskeyReplyBody struct { - Channel string `json:"channel"` - Type string `json:"type"` - Body MisskeyNoteBody `json:"body"` -} - -type MisskeyNoteBody struct { - Uri string `json:"uri"` // Remote Note - Id string `json:"id"` // Local note -} - -type MisskeyRequest struct { - Type string `json:"type"` - Body MisskeyRequestBody `json:"body"` -} - -type MisskeyRequestBody struct { - Channel string `json:"channel"` - Id string `json:"id"` - Params MisskeyRequestParams -} - -type MisskeyRequestParams struct { -} - -func StreamMisskey(endpoint string) { - logDebug("StreamMisskey for ", endpoint) - u := url.URL{ - Scheme: "wss", - Host: endpoint, - Path: "/streaming", - } - - var misskeyrequest MisskeyRequest - misskeyrequest.Type = "connect" - misskeyrequest.Body.Channel = "globalTimeline" - misskeyrequest.Body.Id = uuid.New().String() - - for { - misskeyrequeststream, err := json.Marshal(misskeyrequest) - if err != nil { - panic(err) - } - - for { - // Create a new WebSocket connection. - ws, _, err := websocket.DefaultDialer.Dial(u.String(), nil) - if err != nil { - logErr("Error dialing misskey webSocket to ", endpoint, " :", err) - return - } - logDebug("Misskey websocket connection created: ", endpoint) - - ri_mutex.Lock() - m := runninginstances[endpoint] - m.Status = shared.RUNNING - m.LastRun = "Streaming" - runninginstances[endpoint] = m - ri_mutex.Unlock() - - // Send a message to the server. - err = ws.WriteMessage(websocket.TextMessage, misskeyrequeststream) - if err != nil { - logErr("Error sending misskey channel subscription: ", endpoint) - return - } - logDebug("Successfully sent misskey channel subscription: ", endpoint) - - // Read a message from the server. - for { - logDebug("Starting Misskey Stream loop for ", endpoint) - _, message, err := ws.ReadMessage() - if err != nil { - logErr("Misskey stream broken: ", endpoint) - return - } - - // Print the message to the console. - logDebug("Ending Misskey Stream loop for ", endpoint) - var misskeyreply MisskeyReply - err = json.Unmarshal(message, &misskeyreply) - if err != nil { - logErr("Unable to parse data from "+endpoint+", but still connected, err: ", err) - break - } - // newactivity := misskeyreply.Body.Body - - var newactivity string - - if misskeyreply.Body.Body.Uri != "" { // Remote Message - newactivity = misskeyreply.Body.Body.Uri - matchset := re.FindStringSubmatch(newactivity) - if matchset != nil { - newinstance := matchset[1] - logDebug("Checking new instance from Misskey Stream: ", newinstance) - go CheckInstance(newinstance, endpoint) - } - } else { // Local Message - newactivity = "https://" + endpoint + "/notes/" + misskeyreply.Body.Body.Id - } - - logDebug("Misskey new URI ", newactivity, " from instance: ", endpoint) - - go check_activity(newactivity) - } - // Close the WebSocket connection. - ws.Close() - time.Sleep(time.Minute * 10) - } - } -} - -func StreamPleroma(endpoint string) { - wss_url := "wss://" + endpoint + "/api/v1/streaming/?stream=public" - var pleromaHeader shared.PleromaStreamHeader - var newactivity shared.ReportActivity - var err error - - for { - var tries int - var ws *websocket.Conn - for tries = 0; tries < 10; tries++ { - ws, _, err = websocket.DefaultDialer.Dial(wss_url, nil) - if err != nil { - continue - } - break - } - - if tries == 10 { - logWarn("Unable to connect to " + endpoint + " after 10 tries, exiting") - return - } - - ri_mutex.Lock() - m := runninginstances[endpoint] - m.Status = shared.RUNNING - m.LastRun = "Streaming" - runninginstances[endpoint] = m - ri_mutex.Unlock() - - for { - logDebug("Starting Pleroma Stream loop for ", endpoint) - _, p, err := ws.ReadMessage() - if err != nil { - logErr("Unable to read message from Pleroma stream: ", endpoint, " Err: ", err) - break - } - - err = json.Unmarshal(p, &pleromaHeader) - if err != nil { - logErr("Unable to parse data from "+endpoint+", but still connected, err: ", err) - break - } - - switch pleromaHeader.Event { - case "update": - err = json.Unmarshal([]byte(pleromaHeader.Payload), &newactivity) - if err != nil { - logErr("Unable to parse data from " + endpoint + ", but still connected.") - break - } - go check_activity(newactivity.Uri) - matchset := re.FindStringSubmatch(newactivity.Uri) - if matchset != nil { - newinstance := matchset[1] - logDebug("Checking new instance from Pleroma Stream: ", newinstance) - go CheckInstance(newinstance, endpoint) - } - default: - logDebug("Unimplemented pleroma stream activity: ", pleromaHeader.Event) - continue - } - logDebug("Ending Pleroma stream loop for ", endpoint) - } - ws.Close() - // time.Sleep(time.Minute * 10) - } -} - -func StreamMastodon(endpoint string, o *shared.RunningInstance) { - stream_client := BuildClient(endpoint) - - var retry bool - - api_timeline := "https://" + endpoint + "/api/v1/streaming/public" - - for { - req, err := http.NewRequest("GET", api_timeline, nil) - req.Header.Set("User-Agent", "Tusky") - if err != nil { - logFatal("Unable to create new request for " + endpoint + ", exiting.") - return - } - - var resp *http.Response - - for tries := 0; tries < 10; tries++ { - resp, err = stream_client.Do(req) - if err != nil { - time.Sleep(time.Minute * 5) - logWarn("Failure connecting to "+req.URL.Scheme+"://"+req.URL.Host+req.URL.Path+", attempt ", tries+1, ", sleeping for 5 minutes, ", err) - continue - } - break - } - if err != nil { - logErr("Unable to stream "+api_timeline+": ", err) - return - } - defer resp.Body.Close() - - ri_mutex.Lock() - m := runninginstances[endpoint] - m.Status = shared.RUNNING - m.LastRun = "Streaming" - runninginstances[endpoint] = m - ri_mutex.Unlock() - - s := bufio.NewScanner(resp.Body) - var name string - for s.Scan() { - logDebug("Starting Mastodon stream loop for ", endpoint) - line := s.Text() - token := strings.SplitN(line, ":", 2) - var newactivity shared.ReportActivity - if len(token) != 2 { - continue - } - - switch strings.TrimSpace(token[0]) { - case "event": - name = strings.TrimSpace(token[1]) - continue - case "data": - switch name { - case "update": - if len(token) >= 2 && len(token[1]) >= 2 { - continue - } - jsondata := token[1][1:] - err := json.Unmarshal([]byte(jsondata), &newactivity) - if err != nil { - logDebug("Unable to parse data from " + endpoint + ", but still connected.") - continue - } - retry = true - default: - continue - } - default: - continue - } - - go check_activity(newactivity.Uri) - - matchset := re.FindStringSubmatch(newactivity.Uri) - if matchset != nil { - newinstance := matchset[1] - logDebug("Checking new instance from Mastodon Stream: ", newinstance) - go CheckInstance(newinstance, endpoint) - } - logDebug("Ending Mastodon stream loop for ", endpoint) - } - if retry == true { - time.Sleep(time.Minute * 10) - } else { - break - } - } -} diff --git a/fedilogger/tables.sql b/fedilogger/tables.sql deleted file mode 100644 index 3d17d72..0000000 --- a/fedilogger/tables.sql +++ /dev/null @@ -1,731 +0,0 @@ -CREATE TABLE IF NOT EXISTS actors ( - id SERIAL PRIMARY KEY, - document JSONB, - identifiedat TIMESTAMP with time zone DEFAULT now(), - instance VARCHAR(1000) NOT NULL, - bot BOOLEAN DEFAULT FALSE -); - -CREATE TABLE IF NOT EXISTS activities ( - id SERIAL PRIMARY KEY, - document JSONB, - normalized TEXT, - identifiedat TIMESTAMP with time zone DEFAULT now(), - instance VARCHAR(1000) NOT NULL, - hashtags VARCHAR(140)[], - bot BOOLEAN DEFAULT FALSE -); - -CREATE TABLE IF NOT EXISTS instances ( - endpoint VARCHAR(2083) NOT NULL PRIMARY KEY UNIQUE, - state VARCHAR(16), - username VARCHAR(32), - password VARCHAR(32), - software VARCHAR(50), - banned BOOLEAN DEFAULT FALSE, - alwaysbot BOOLEAN DEFAULT FALSE -); - --- Autostart mastodon.social -INSERT INTO instances (endpoint) VALUES('mastodon.social'); --- Banned Instances -INSERT INTO instances (endpoint, banned) VALUES - ('switter.at', true), - ('xxxtumblr.org', true), - ('sinblr.com', true), - ('twitiverse.com', true), - ('my.dirtyhobby.xyz', true), - ('bae.st', true); --- Alwaysbot instances -INSERT INTO instances (endpoint, alwaysbot) VALUES - ('mstdn.foxfam.club', true), - ('botsin.space', true), - ('newsbots.eu', true); - -ALTER TABLE activities - ADD normalized_tsvector tsvector - GENERATED ALWAYS AS (to_tsvector('english', normalized)) STORED; - -CREATE UNIQUE INDEX IF NOT EXISTS actors_uri_idx ON actors ( (document->>'id') ); -CREATE UNIQUE INDEX IF NOT EXISTS activities_uri_idx ON activities ( (document->>'id') ); - -CREATE INDEX IF NOT EXISTS activities_published_idx ON activities ( (document->>'published') ); -CREATE INDEX IF NOT EXISTS activities_identifiedat_idx ON activities (identifiedat); -CREATE INDEX IF NOT EXISTS hashtags_idx ON activities(hashtags); - -CREATE INDEX IF NOT EXISTS normalized_idx ON activities USING gin(normalized_tsvector); - -CREATE INDEX IF NOT EXISTS actors_id_idx ON actors (id); -CREATE INDEX IF NOT EXISTS activities_id_idx ON activities (id); - -CREATE TABLE IF NOT EXISTS stopwords ( - word VARCHAR(20) -); - -INSERT INTO stopwords (word) -VALUES - ('a'), - ('able'), - ('about'), - ('above'), - ('abst'), - ('accordance'), - ('according'), - ('accordingly'), - ('across'), - ('act'), - ('actually'), - ('added'), - ('adj'), - ('affected'), - ('affecting'), - ('affects'), - ('after'), - ('afterwards'), - ('again'), - ('against'), - ('ah'), - ('all'), - ('almost'), - ('alone'), - ('along'), - ('already'), - ('also'), - ('although'), - ('always'), - ('am'), - ('among'), - ('amongst'), - ('an'), - ('and'), - ('announce'), - ('another'), - ('any'), - ('anybody'), - ('anyhow'), - ('anymore'), - ('anyone'), - ('anything'), - ('anyway'), - ('anyways'), - ('anywhere'), - ('apparently'), - ('approximately'), - ('are'), - ('aren'), - ('arent'), - ('arise'), - ('around'), - ('as'), - ('aside'), - ('ask'), - ('asking'), - ('at'), - ('auth'), - ('available'), - ('away'), - ('awfully'), - ('b'), - ('back'), - ('be'), - ('became'), - ('because'), - ('become'), - ('becomes'), - ('becoming'), - ('been'), - ('before'), - ('beforehand'), - ('begin'), - ('beginning'), - ('beginnings'), - ('begins'), - ('behind'), - ('being'), - ('believe'), - ('below'), - ('beside'), - ('besides'), - ('between'), - ('beyond'), - ('biol'), - ('both'), - ('brief'), - ('briefly'), - ('but'), - ('by'), - ('c'), - ('ca'), - ('came'), - ('can'), - ('cannot'), - ('can''t'), - ('cause'), - ('causes'), - ('certain'), - ('certainly'), - ('co'), - ('com'), - ('come'), - ('comes'), - ('contain'), - ('containing'), - ('contains'), - ('could'), - ('couldnt'), - ('d'), - ('date'), - ('did'), - ('didn''t'), - ('different'), - ('do'), - ('does'), - ('doesn''t'), - ('doing'), - ('done'), - ('don''t'), - ('down'), - ('downwards'), - ('due'), - ('during'), - ('e'), - ('each'), - ('ed'), - ('edu'), - ('effect'), - ('eg'), - ('eight'), - ('eighty'), - ('either'), - ('else'), - ('elsewhere'), - ('end'), - ('ending'), - ('enough'), - ('especially'), - ('et'), - ('et-al'), - ('etc'), - ('even'), - ('ever'), - ('every'), - ('everybody'), - ('everyone'), - ('everything'), - ('everywhere'), - ('ex'), - ('except'), - ('f'), - ('far'), - ('few'), - ('ff'), - ('fifth'), - ('first'), - ('five'), - ('fix'), - ('followed'), - ('following'), - ('follows'), - ('for'), - ('former'), - ('formerly'), - ('forth'), - ('found'), - ('four'), - ('from'), - ('further'), - ('furthermore'), - ('g'), - ('gave'), - ('get'), - ('gets'), - ('getting'), - ('give'), - ('given'), - ('gives'), - ('giving'), - ('go'), - ('goes'), - ('gone'), - ('got'), - ('gotten'), - ('h'), - ('had'), - ('happens'), - ('hardly'), - ('has'), - ('hasn''t'), - ('have'), - ('haven''t'), - ('having'), - ('he'), - ('hed'), - ('hence'), - ('her'), - ('here'), - ('hereafter'), - ('hereby'), - ('herein'), - ('heres'), - ('hereupon'), - ('hers'), - ('herself'), - ('hes'), - ('hi'), - ('hid'), - ('him'), - ('himself'), - ('his'), - ('hither'), - ('home'), - ('how'), - ('howbeit'), - ('however'), - ('hundred'), - ('i'), - ('id'), - ('ie'), - ('if'), - ('i''ll'), - ('im'), - ('immediate'), - ('immediately'), - ('importance'), - ('important'), - ('in'), - ('inc'), - ('indeed'), - ('index'), - ('information'), - ('instead'), - ('into'), - ('invention'), - ('inward'), - ('is'), - ('isn''t'), - ('it'), - ('itd'), - ('it''ll'), - ('its'), - ('itself'), - ('i''ve'), - ('j'), - ('just'), - ('k'), - ('keep keeps'), - ('kept'), - ('kg'), - ('km'), - ('know'), - ('known'), - ('knows'), - ('l'), - ('largely'), - ('last'), - ('lately'), - ('later'), - ('latter'), - ('latterly'), - ('least'), - ('less'), - ('lest'), - ('let'), - ('lets'), - ('like'), - ('liked'), - ('likely'), - ('line'), - ('little'), - ('''ll'), - ('look'), - ('looking'), - ('looks'), - ('ltd'), - ('m'), - ('made'), - ('mainly'), - ('make'), - ('makes'), - ('many'), - ('may'), - ('maybe'), - ('me'), - ('mean'), - ('means'), - ('meantime'), - ('meanwhile'), - ('merely'), - ('mg'), - ('might'), - ('million'), - ('miss'), - ('ml'), - ('more'), - ('moreover'), - ('most'), - ('mostly'), - ('mr'), - ('mrs'), - ('much'), - ('mug'), - ('must'), - ('my'), - ('myself'), - ('n'), - ('na'), - ('name'), - ('namely'), - ('nay'), - ('nd'), - ('near'), - ('nearly'), - ('necessarily'), - ('necessary'), - ('need'), - ('needs'), - ('neither'), - ('never'), - ('nevertheless'), - ('new'), - ('next'), - ('nine'), - ('ninety'), - ('no'), - ('nobody'), - ('non'), - ('none'), - ('nonetheless'), - ('noone'), - ('nor'), - ('normally'), - ('nos'), - ('not'), - ('noted'), - ('nothing'), - ('now'), - ('nowhere'), - ('o'), - ('obtain'), - ('obtained'), - ('obviously'), - ('of'), - ('off'), - ('often'), - ('oh'), - ('ok'), - ('okay'), - ('old'), - ('omitted'), - ('on'), - ('once'), - ('one'), - ('ones'), - ('only'), - ('onto'), - ('or'), - ('ord'), - ('other'), - ('others'), - ('otherwise'), - ('ought'), - ('our'), - ('ours'), - ('ourselves'), - ('out'), - ('outside'), - ('over'), - ('overall'), - ('owing'), - ('own'), - ('p'), - ('page'), - ('pages'), - ('part'), - ('particular'), - ('particularly'), - ('past'), - ('per'), - ('perhaps'), - ('placed'), - ('please'), - ('plus'), - ('poorly'), - ('possible'), - ('possibly'), - ('potentially'), - ('pp'), - ('predominantly'), - ('present'), - ('previously'), - ('primarily'), - ('probably'), - ('promptly'), - ('proud'), - ('provides'), - ('put'), - ('q'), - ('que'), - ('quickly'), - ('quite'), - ('qv'), - ('r'), - ('ran'), - ('rather'), - ('rd'), - ('re'), - ('readily'), - ('really'), - ('recent'), - ('recently'), - ('ref'), - ('refs'), - ('regarding'), - ('regardless'), - ('regards'), - ('related'), - ('relatively'), - ('research'), - ('respectively'), - ('resulted'), - ('resulting'), - ('results'), - ('right'), - ('run'), - ('s'), - ('said'), - ('same'), - ('saw'), - ('say'), - ('saying'), - ('says'), - ('sec'), - ('section'), - ('see'), - ('seeing'), - ('seem'), - ('seemed'), - ('seeming'), - ('seems'), - ('seen'), - ('self'), - ('selves'), - ('sent'), - ('seven'), - ('several'), - ('shall'), - ('she'), - ('shed'), - ('she''ll'), - ('shes'), - ('should'), - ('shouldn''t'), - ('show'), - ('showed'), - ('shown'), - ('showns'), - ('shows'), - ('significant'), - ('significantly'), - ('similar'), - ('similarly'), - ('since'), - ('six'), - ('slightly'), - ('so'), - ('some'), - ('somebody'), - ('somehow'), - ('someone'), - ('somethan'), - ('something'), - ('sometime'), - ('sometimes'), - ('somewhat'), - ('somewhere'), - ('soon'), - ('sorry'), - ('specifically'), - ('specified'), - ('specify'), - ('specifying'), - ('still'), - ('stop'), - ('strongly'), - ('sub'), - ('substantially'), - ('successfully'), - ('such'), - ('sufficiently'), - ('suggest'), - ('sup'), - ('sure t'), - ('take'), - ('taken'), - ('taking'), - ('tell'), - ('tends'), - ('th'), - ('than'), - ('thank'), - ('thanks'), - ('thanx'), - ('that'), - ('that''ll'), - ('thats'), - ('that''ve'), - ('the'), - ('their'), - ('theirs'), - ('them'), - ('themselves'), - ('then'), - ('thence'), - ('there'), - ('thereafter'), - ('thereby'), - ('thered'), - ('therefore'), - ('therein'), - ('there''ll'), - ('thereof'), - ('therere'), - ('theres'), - ('thereto'), - ('thereupon'), - ('there''ve'), - ('these'), - ('they'), - ('theyd'), - ('they''ll'), - ('theyre'), - ('they''ve'), - ('think'), - ('this'), - ('those'), - ('thou'), - ('though'), - ('thoughh'), - ('thousand'), - ('throug'), - ('through'), - ('throughout'), - ('thru'), - ('thus'), - ('til'), - ('tip'), - ('to'), - ('together'), - ('too'), - ('took'), - ('toward'), - ('towards'), - ('tried'), - ('tries'), - ('truly'), - ('try'), - ('trying'), - ('ts'), - ('twice'), - ('two'), - ('u'), - ('un'), - ('under'), - ('unfortunately'), - ('unless'), - ('unlike'), - ('unlikely'), - ('until'), - ('unto'), - ('up'), - ('upon'), - ('ups'), - ('us'), - ('use'), - ('used'), - ('useful'), - ('usefully'), - ('usefulness'), - ('uses'), - ('using'), - ('usually'), - ('v'), - ('value'), - ('various'), - ('''ve'), - ('very'), - ('via'), - ('viz'), - ('vol'), - ('vols'), - ('vs'), - ('w'), - ('want'), - ('wants'), - ('was'), - ('wasnt'), - ('way'), - ('we'), - ('wed'), - ('welcome'), - ('we''ll'), - ('went'), - ('were'), - ('werent'), - ('we''ve'), - ('what'), - ('whatever'), - ('what''ll'), - ('whats'), - ('when'), - ('whence'), - ('whenever'), - ('where'), - ('whereafter'), - ('whereas'), - ('whereby'), - ('wherein'), - ('wheres'), - ('whereupon'), - ('wherever'), - ('whether'), - ('which'), - ('while'), - ('whim'), - ('whither'), - ('who'), - ('whod'), - ('whoever'), - ('whole'), - ('who''ll'), - ('whom'), - ('whomever'), - ('whos'), - ('whose'), - ('why'), - ('widely'), - ('willing'), - ('wish'), - ('with'), - ('within'), - ('without'), - ('wont'), - ('words'), - ('world'), - ('would'), - ('wouldnt'), - ('www'), - ('x'), - ('y'), - ('yes'), - ('yet'), - ('you'), - ('youd'), - ('you''ll'), - ('your'), - ('youre'), - ('yours'), - ('yourself'), - ('yourselves'), - ('you''ve'), - ('z'), - ('zero'); diff --git a/fedilogger/testhelper.go b/fedilogger/testhelper.go deleted file mode 100644 index 9baca3a..0000000 --- a/fedilogger/testhelper.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -import ( - "reflect" - "testing" -) - -// AssertEqual checks if values are equal -func AssertEqual(t *testing.T, a interface{}, b interface{}) { - if a == b { - return - } - // debug.PrintStack() - t.Errorf("Received %v (type %v), expected %v (type %v)", a, reflect.TypeOf(a), b, reflect.TypeOf(b)) -} diff --git a/fedilogger/web.go b/fedilogger/web.go deleted file mode 100644 index cf2d028..0000000 --- a/fedilogger/web.go +++ /dev/null @@ -1,311 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "strings" -) - -// CreateObject - Used by post web receiver -type CreateObject struct { - ID string `json:"id"` - Actor string `json:"actor"` - Cc []string `json:"cc"` - Content string `json:"content"` - To []string `json:"to"` - Type string `json:"type"` -} - -// CreateObject - Used by post web receiver -type AnnounceObject struct { - ID string `json:"id"` - Actor string `json:"actor"` - To []string `json:"to"` - Type string `json:"type"` - Object string `json:"object"` -} - -// RelayBase - The base object used by web receiver -type RelayBase struct { - Actor string `json:"actor"` - Cc []string `json:"cc"` - Object json.RawMessage `json:"Object"` - ID string `json:"id"` - Published string `json:"published"` - To []string `json:"to"` - Type string `json:"type"` -} - -func hostmeta(w http.ResponseWriter, r *http.Request) { - fmt.Println("PATH --> ", r.URL.Path) - host := r.Host - xml := "" - w.Header().Set("Content-Type", "application/xrd+xml") - fmt.Fprintf(w, xml) -} - -func webfinger(w http.ResponseWriter, r *http.Request) { - fmt.Println("PATH --> ", r.URL.Path) - host := r.Host - - webfingermap := make(map[string]interface{}) - webfingermap["subject"] = "acct:fedilogue@" + host - webfingermap["aliases"] = []string{"https://" + host + "/users/fedilogue"} - link0 := make(map[string]string) - link0["rel"] = "http://webfinger.net/rel/profile-page" - link0["type"] = "text/html" - link0["href"] = "https://" + host + "/users/fedilogue" - link1 := make(map[string]string) - link1["rel"] = "self" - link1["type"] = "application/activity+json" - link1["href"] = "https://" + host + "/users/fedilogue" - link2 := make(map[string]string) - link2["rel"] = "self" - link2["type"] = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" - link2["href"] = "https://" + host + "/users/fedilogue" - link3 := make(map[string]string) - link3["rel"] = "http://ostatus.org/schema/1.0/subscribe" - link3["template"] = "https://" + host + "/ostatus_subscribe?acct={uri}" - - links := []map[string]string{link0, link1, link2, link3} - webfingermap["links"] = links - - webfingerbin, err := json.Marshal(webfingermap) - if err != nil { - fmt.Println(err.Error()) - return - } - - webfingerstr := string(webfingerbin) - - query := r.URL.Query() - resourceRaw, exists := query["resource"] - - if exists { - resource := resourceRaw[0] - if resource != "acct:fedilogue@"+host { - fmt.Println("Writes properly but wrong acct") - w.Header().Set("Content-Type", "application/json; charset=utf-8") - fmt.Fprintf(w, webfingerstr) - return - } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - fmt.Fprintf(w, webfingerstr) - } else { - fmt.Println(query) - w.WriteHeader(http.StatusNotFound) - return - } -} - -func inboxHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - - body, err := ioutil.ReadAll(r.Body) - if err != nil { - fmt.Println(err) - return - } - defer r.Body.Close() - - var findtype RelayBase - err = json.Unmarshal(body, &findtype) - if err != nil { - logWarn("Unable to unmarshal here, exiting...") - return - } - - switch findtype.Type { - case "Create": - var createobject CreateObject - - err = json.Unmarshal(body, &createobject) - if err != nil { - return - } - - go check_activity(createobject.ID) - slashend := strings.Index(createobject.ID[8:], "/") - newinstance := createobject.ID[8 : 8+slashend] - - go CheckInstance(newinstance, "") - - case "Update": - case "Reject": - case "Add": - case "Remove": - case "Follow": - case "Accept": - case "Like": - case "Announce": - var announceobject AnnounceObject - err = json.Unmarshal(body, &announceobject) - if err != nil { - fmt.Println(err.Error()) - return - } - check_activity(announceobject.Object) - - matchset := re.FindStringSubmatch(announceobject.Object) - if matchset != nil { - newinstance := matchset[1] - go CheckInstance(newinstance, "") - } - case "Delete": - case "Undo": - default: - logWarn("Unknown ActivityPub request:", findtype.Type) - } - - w.Header().Set("Content-Type", "application/json") - fmt.Fprintf(w, "{}") - } -} - -func internalFetch(w http.ResponseWriter, r *http.Request) { - - publicKeyBytes, err := ioutil.ReadFile("keys/public.pem") - publicKeyString := string(publicKeyBytes) - publicKeyString = strings.Replace(publicKeyString, "\n", "\\n", -11) - if err != nil { - log.Fatal(err) - } - - staticjson := fmt.Sprintf(`{"@context":["https://www.w3.org/ns/activitystreams","https://%[1]s/schemas/litepub-0.1.jsonld",{"@language":"und"}],"endpoints":{"oauthAuthorizationEndpoint":"https://%[1]s/oauth/authorize","oauthRegistrationEndpoint":"https://%[1]s/api/v1/apps","oauthTokenEndpoint":"https://%[1]s/oauth/token","sharedInbox":"https://%[1]s/inbox","uploadMedia":"https://%[1]s/api/ap/upload_media"},"followers":"https://%[1]s/internal/fetch/followers","following":"https://%[1]s/internal/fetch/following","id":"https://%[1]s/internal/fetch","inbox":"https://%[1]s/internal/fetch/inbox","invisible":true,"manuallyApprovesFollowers":false,"name":"Pleroma","preferredUsername":"fedilogue","publicKey":{"id":"https://%[1]s/internal/fetch#main-key","owner":"https://%[1]s/internal/fetch","publicKeyPem":"%[2]s"},"summary":"Fedilogue Key or something.","type":"Application","url":"https://%[1]s/internal/fetch"}`, r.Host, publicKeyString) - - fmt.Fprintf(w, staticjson) - -} - -func relay(w http.ResponseWriter, r *http.Request) { - - fmt.Println("Someone out there requested /relay") - - publicKeyBytes, err := ioutil.ReadFile("keys/public.pem") - publicKeyString := string(publicKeyBytes) - publicKeyString = strings.Replace(publicKeyString, "\n", "\\n", -11) - if err != nil { - log.Fatal(err) - } - - staticjson := fmt.Sprintf(`{"@context":["https://www.w3.org/ns/activitystreams","https://%[1]s/schemas/litepub-0.1.jsonld",{"@language":"und"}],"alsoKnownAs":[],"attachment":[],"capabilities":{},"discoverable":false,"endpoints":{"oauthAuthorizationEndpoint":"https://%[1]s/oauth/authorize","oauthRegistrationEndpoint":"https://%[1]s/api/v1/apps","oauthTokenEndpoint":"https://%[1]s/oauth/token","sharedInbox":"https://%[1]s/inbox","uploadMedia":"https://%[1]s/api/ap/upload_media"},"featured":"https://%[1]s/relay/collections/featured","followers":"https://%[1]s/relay/followers","following":"https://%[1]s/relay/following","id":"https://%[1]s/relay","inbox":"https://%[1]s/relay/inbox","manuallyApprovesFollowers":false,"name":null,"outbox":"https://%[1]s/relay/outbox","preferredUsername":"relay","publicKey":{"id":"https://%[1]s/relay#main-key","owner":"https://%[1]s/relay","publicKeyPem":"%[2]s"},"summary":"","tag":[],"type":"Person","url":"https://%[1]s/relay"}`, r.Host, publicKeyString) - - fmt.Fprintf(w, staticjson) - -} - -func usersFedilogueFollowers(w http.ResponseWriter, r *http.Request) { - fmt.Println("PATH --> ", r.URL.Path) - host := r.Host - contextlist := map[string]string{"@language": "und"} - - context := []interface{}{"https://www.w3.org/ns/activitystreams", "https://" + host + "/schemas/litepub-0.1.jsonld", contextlist} - followersmap := make(map[string]interface{}) - followersmap["@context"] = context - - staticjson := "{\"@context\":[\"https://www.w3.org/ns/activitystreams\",\"https://" + host + "/schemas/litepub-0.1.jsonld\",{\"@language\":\"und\"}],\"first\":{\"id\":\"https://" + host + "/users/fedilogue/followers?page=1\",\"next\":\"https://" + host + "/users/fedilogue/followers?page=2\",\"orderedItems\":[\"https://mastodon.host/users/federationbot\"],\"partOf\":\"https://" + host + "/users/fedilogue/followers\",\"totalItems\":1,\"type\":\"OrderedCollectionPage\"},\"id\":\"https://" + host + "/users/fedilogue/followers\",\"totalItems\":1,\"type\":\"OrderedCollection\"}" - - w.Header().Set("Content-Type", "application/activity+json; charset=utf-8") - fmt.Fprintf(w, staticjson) -} - -func usersFedilogueFollowing(w http.ResponseWriter, r *http.Request) { - host := r.Host - fmt.Println("PATH --> ", r.URL.Path) - staticjson := "{\"@context\": [\"https://www.w3.org/ns/activitystreams\", \"https://" + host + "/schemas/litepub-0.1.jsonld\", {\"@language\": \"und\"}], \"first\": {\"id\": \"https://" + host + "/users/fedilogue/following?page=1\", \"orderedItems\": [], \"partOf\": \"https://" + host + "/users/fedilogue/following\", \"totalItems\": 0, \"type\": \"OrderedCollectionPage\"}, \"id\": \"https://" + host + "/users/fedilogue/following\", \"totalItems\": 0, \"type\": \"OrderedCollection\"}" - w.Header().Set("Content-Type", "application/activity+json; charset=utf-8") - fmt.Fprintf(w, staticjson) -} - -func usersFedilogue(w http.ResponseWriter, r *http.Request) { - fmt.Println("PATH --> ", r.URL.Path) - host := r.Host - - fmt.Println(r.Host) - fmt.Println(r.Header["Accept"]) - - publickeybin, err := ioutil.ReadFile("keys/public.pem") - if err != nil { - fmt.Println(err) - os.Exit(1) - } - - publickeypemstr := string(publickeybin) - - publicKey := map[string]string{"id": "https://" + host + "/users/fedilogue#main-key", "owner": "https://" + host + "/users/fedilogue", "publicKeyPem": publickeypemstr} - - capabilities := map[string]bool{} - tag := []string{} - contextlist := map[string]string{"@language": "und"} - attachment := []string{} - endpoints := map[string]string{"oauthAuthorizationEndpoint": "https://" + host + "/oauth/authorize", "oauthRegistrationEndpoint": "https://" + host + "/api/v1/apps", "oauthTokenEndpoint": "https://" + host + "/oauth/token", "sharedInbox": "https://" + host + "/inbox", "uploadMedia": "https://" + host + "/api/ap/upload_media"} - - context := []interface{}{"https://www.w3.org/ns/activitystreams", "https://" + host + "/schemas/litepub-0.1.jsonld", contextlist} - actorjsonmap := make(map[string]interface{}) - actorjsonmap["@context"] = context - actorjsonmap["attachment"] = attachment - actorjsonmap["capabilities"] = capabilities - actorjsonmap["discoverable"] = false - actorjsonmap["endpoints"] = endpoints - actorjsonmap["followers"] = "https://" + host + "/users/fedilogue/followers" - actorjsonmap["following"] = "https://" + host + "/users/fedilogue/following" - actorjsonmap["id"] = "https://" + host + "/users/fedilogue" - actorjsonmap["inbox"] = "https://" + host + "/users/fedilogue/inbox" - actorjsonmap["manuallyApprovesFollowers"] = false - actorjsonmap["name"] = "Fedilogue Mass Follower" - actorjsonmap["outbox"] = "https://" + host + "/users/fedilogue/outbox" - actorjsonmap["preferredUsername"] = "fedilogue" - actorjsonmap["publicKey"] = publicKey - actorjsonmap["summary"] = "" - actorjsonmap["tag"] = tag - actorjsonmap["type"] = "Application" - actorjsonmap["uri"] = "https://" + host + "/users/fedilogue" - - actorjsonbin, err := json.Marshal(actorjsonmap) - if err != nil { - fmt.Println(err.Error()) - return - } - - actorjsonstr := string(actorjsonbin) - w.Header().Set("Content-Type", "application/activity+json; charset=utf-8") - fmt.Fprintf(w, actorjsonstr) -} - -func errorHandler(w http.ResponseWriter, r *http.Request) { - for name, headers := range r.Header { - fmt.Println("ROW: ", name, " ", headers) - for _, h := range headers { - fmt.Fprintf(w, "%v: %v\n", name, h) - } - } - - reqBody, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Fatal(err) - } - - fmt.Printf("POST DATA: %s\n", reqBody) - -} - -func webmain() { - - webreceiver := http.NewServeMux() - - webreceiver.HandleFunc("/.well-known/webfinger", webfinger) - webreceiver.HandleFunc("/.well-known/host-meta", hostmeta) - webreceiver.HandleFunc("/inbox", inboxHandler()) - webreceiver.HandleFunc("/internal/fetch", internalFetch) - webreceiver.HandleFunc("/relay", relay) - webreceiver.HandleFunc("/users/fedilogue", usersFedilogue) - webreceiver.HandleFunc("/users/fedilogue/followers", usersFedilogueFollowers) - webreceiver.HandleFunc("/users/fedilogue/following", usersFedilogueFollowing) - webreceiver.HandleFunc("/", errorHandler) - log.Print("Starting HTTP inbox on port 127.0.0.1:8042") - log.Fatal(http.ListenAndServe("127.0.0.1:8042", webreceiver)) -} diff --git a/fedilogue/Dockerfile b/fedilogue/Dockerfile new file mode 100644 index 0000000..4eabf7a --- /dev/null +++ b/fedilogue/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:latest AS build +WORKDIR /build +COPY go.mod go.sum . +RUN go mod download +COPY . . +RUN go build . + +FROM alpine:latest AS run +WORKDIR /app +RUN apk add gcompat +COPY --from=build /build/fedilogue /app/fedilogue +ENTRYPOINT ["/app/fedilogue"] diff --git a/restapi/Dockerfile b/restapi/Dockerfile index c275816..6bc22ea 100644 --- a/restapi/Dockerfile +++ b/restapi/Dockerfile @@ -5,7 +5,8 @@ RUN go mod download COPY . . RUN go build . -FROM alpine:latest +FROM alpine:latest AS run WORKDIR /app +RUN apk add gcompat COPY --from=build /build/restapi /app/restapi ENTRYPOINT ["/app/restapi"]