diff --git a/fedilogue/config.go b/fedilogue/config.go new file mode 100644 index 0000000..6fddd32 --- /dev/null +++ b/fedilogue/config.go @@ -0,0 +1,37 @@ +package main + +import ( + "flag" + "fmt" + +) + +// Settings - Configuration file structure +type Settings struct { + Crawl bool + LogLevel int + Hostname string +} + +var settings Settings + +/* Test: TestStringexists */ +func stringexists(needle string, haystack []string) bool { + for _, check := range haystack { + if check == needle { + return true + } + } + return false +} + +func getSettings() { + flag.BoolVar(&settings.Crawl, "c", true, "Crawl mode (default is yes)") + flag.StringVar(&settings.Hostname, "h", "myhostname", "Set your hostname") + flag.IntVar(&settings.LogLevel, "l", 1, "Logging Level:\n 0) No logs\n 1) Reports every 30 seconds\n 2) Errors\n 3) Warnings\n 4) New Connections\n 5) Debugging\n") + + fmt.Println("Crawl: ", settings.Crawl) + + flag.Parse() + fmt.Println(settings) +} diff --git a/fedilogue/config_test.go b/fedilogue/config_test.go new file mode 100644 index 0000000..40369a8 --- /dev/null +++ b/fedilogue/config_test.go @@ -0,0 +1,14 @@ +package main + +import ( + "testing" +) + +func TestStringexists(t *testing.T) { + var empty_strings = []string {} + var three_strings = []string {"first", "second", "third"} + + AssertEqual(t, stringexists("amything", empty_strings), false) + AssertEqual(t, stringexists("second", three_strings), true) + AssertEqual(t, stringexists("fourth", three_strings), false) +} diff --git a/fedilogue/ctl.go b/fedilogue/ctl.go new file mode 100644 index 0000000..6dfd171 --- /dev/null +++ b/fedilogue/ctl.go @@ -0,0 +1,71 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" +) + +func cmdFollow(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + fmt.Println(err) + return + } + defer r.Body.Close() + + var followtarget interface{} + err = json.Unmarshal(body, &followtarget) + if err != nil { + fmt.Println(err) + return + } + + myMap := followtarget.(map[string]interface{}) + followInbox(myMap["follow"].(string)) + + fmt.Fprintf(w, "{}") +} + +func cmdAdd(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + fmt.Println(err) + return + } + defer r.Body.Close() + + var addtarget interface{} + err = json.Unmarshal(body, &addtarget) + if err != nil { + fmt.Println(err) + return + } + + myMap := addtarget.(map[string]interface{}) + go StartInstance(myMap["host"].(string)) + + fmt.Fprintf(w, "{}") +} + +func cmdStatus(w http.ResponseWriter, r *http.Request) { + ri_mutex.Lock() + bytes, err := json.Marshal(runninginstances) + if err != nil { + fmt.Println("Error happened", err) + } + ri_mutex.Unlock() + fmt.Fprintf(w, (string(bytes))) +} + +func startctl() { + ctlweb := http.NewServeMux() + ctlweb.HandleFunc("/follow", cmdFollow) + ctlweb.HandleFunc("/status", cmdStatus) + ctlweb.HandleFunc("/add", cmdAdd) + log.Print("Starting HTTP inbox on port 127.0.0.1:5555") + log.Fatal(http.ListenAndServe("127.0.0.1:5555", ctlweb)) + +} diff --git a/fedilogue/db.go b/fedilogue/db.go new file mode 100644 index 0000000..c121f25 --- /dev/null +++ b/fedilogue/db.go @@ -0,0 +1,20 @@ +package main + +import ( + "context" + "os" + + "github.com/jackc/pgx/v4/pgxpool" +) + +var pool *pgxpool.Pool + +func getDbPool() *pgxpool.Pool { + // Setup Database + dburl := "postgres://" + os.Getenv("POSTGRES_USER") + ":" + os.Getenv("POSTGRES_PASSWORD") + "@" + os.Getenv("POSTGRES_HOSTNAME") + "/" + os.Getenv("POSTGRES_DB") + pool, err := pgxpool.Connect(context.Background(), dburl) + if err != nil { + logFatal("Unable to connect to database:", err) + } + return pool +} diff --git a/fedilogue/fedilogue.go b/fedilogue/fedilogue.go new file mode 100644 index 0000000..83bf9e4 --- /dev/null +++ b/fedilogue/fedilogue.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "net/http" + _ "net/http/pprof" + "regexp" + "runtime" + "sync" + "time" + + "github.com/microcosm-cc/bluemonday" + "git.farhan.codes/farhan/fedilogue/shared" +) + +// Current instances +var runninginstances map[string]shared.RunningInstance +var ri_mutex = &sync.Mutex{} + +func startpprof() { + logInfo("Starting http/pprof on :7777") + logFatal(http.ListenAndServe("127.0.0.1:7777", nil)) +} + +func statusReportHandler() { + for { + StatusReport() + time.Sleep(time.Second * 60) + } +} + +/* Tests: + - TestStatusReport_empty_run + - TestStatusReport_full_content +*/ +func StatusReport() { + running := 0 + keepalive := 0 + unsupported := 0 + + mastodon := 0 + pleroma := 0 + misskey := 0 + other := 0 + ri_mutex.Lock() + for i, o := range runninginstances { + logDebug("Software ", o.Software, " Status: ", o.Status, " instance ", i) + if o.Status == 200 { + running = running + 1 + } else if o.Status == 607 { // Keepalive + keepalive = keepalive + 1 + } else if o.Status == 605 { // Unsupported instance + unsupported = unsupported + 1 + } + + if o.Software == "mastodon" && o.Status == 200 { + mastodon = mastodon + 1 + } else if o.Software == "pleroma" && o.Status == 200 { + pleroma = pleroma + 1 + } else if o.Software == "misskey" && o.Status == 200 { + misskey = misskey + 1 + } else if o.Status == 200 { + other = other + 1 + } + } + ri_mutex.Unlock() + logInfo("Running:", running, " Keepalive:", keepalive, " Unsupported:", unsupported, " Ma:", mastodon, ",P:", pleroma, ",Mi:", misskey, ",O:", other) +} + +func main() { + // Initial Setup + logInit() + runninginstances = make(map[string]shared.RunningInstance) + + getSettings() + go startpprof() + + pool = getDbPool() + + p = bluemonday.NewPolicy() + spaceReg = regexp.MustCompile(`[\s\t\.]+`) + removeHTMLReg = regexp.MustCompile(`<\/?\s*br\s*>`) + re = regexp.MustCompile("^https?://([^/]*)/(.*)$") + matchurl = regexp.MustCompile("http?s://[\\w\\-]+\\.[\\w\\-]+\\S*") + staggeredStartChan = make(chan bool) + + // Start instances located in database + rows, err := pool.Query(context.Background(), "SELECT endpoint FROM instances") + if err != nil { + logErr("Unable to select from instances") + return + } + defer rows.Close() + + go staggeredStart() + go statusReportHandler() + + for rows.Next() { + var endpoint string + err = rows.Scan(&endpoint) + if err != nil { + logErr("Unable to iterate database, exiting.") + return + } + o, exists := GetRunner(endpoint) + if o.Banned == true { + continue // Banned instance + } + if exists == false { + go StartInstance(endpoint) + } + } + + go startctl() + go webmain() + + runtime.Goexit() +} diff --git a/fedilogue/fedilogue_test.go b/fedilogue/fedilogue_test.go new file mode 100644 index 0000000..bf36b81 --- /dev/null +++ b/fedilogue/fedilogue_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "strconv" + "testing" + "time" + + "git.farhan.codes/farhan/fedilogue/shared" +) + +func TestStatusReport_empty_run(t *testing.T) { + // Empty Instances run + StatusReport() +} + +func TestStatusReport_full_content(t *testing.T) { + defer func() { + runninginstances = map[string]shared.RunningInstance{} + }() + runninginstances = make(map[string]shared.RunningInstance) + + identifier := 0 + var endpoint string + + test_instance_types := []string{"pleroma", "mastodon", "unknown", ""} + test_statuses := []int{shared.NEW_INSTANCE, shared.RUNNING, shared.UNAUTHORIZED, shared.FORBIDDEN, shared.NOT_FOUND, shared.UNPROCESSABLE_ENTITY, shared.TOOMANYREQUESTS, shared.INTERNAL_ERROR, shared.CLIENT_ISSUE, shared.ONION_PROTOCOL, shared.BAD_RESPONSE, shared.BAD_NODEINFO, shared.UNSUPPORTED_INSTANCE, shared.STREAM_ENDED, shared.KEEPALIVE} + + for _, test_instance_type := range test_instance_types { + for _, test_status := range test_statuses { + a := shared.RunningInstance{} + endpoint = "endpoint" + strconv.Itoa(identifier) + ".test.com" + a.Client = BuildClient(endpoint) + a.Status = test_status + a.Recentactivities = shared.NewUniqueFifo(10) + a.Recentactors = shared.NewUniqueFifo(10) + a.Software = test_instance_type + a.Version = "0." + strconv.Itoa(identifier) + a.LastRun = time.Now().Format(time.RFC3339) + runninginstances[endpoint] = a + identifier = identifier + 1 + } + } + + StatusReport() + +} diff --git a/fedilogue/follow.go b/fedilogue/follow.go new file mode 100644 index 0000000..f3a462a --- /dev/null +++ b/fedilogue/follow.go @@ -0,0 +1,85 @@ +package main + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + + "encoding/pem" + "fmt" + "io/ioutil" + "log" + "net/http" + "strings" + "time" + + "github.com/go-fed/httpsig" +) + +func followInbox(inboxurl string) { + matchset := re.FindStringSubmatch(inboxurl) + if matchset == nil { + logWarn("Unable to unregex request", inboxurl) + return + } + inboxhostname := matchset[1] + + keyBytes, err := ioutil.ReadFile("keys/private.pem") + if err != nil { + log.Fatal(err) + } + + pemBlock, _ := pem.Decode(keyBytes) + if pemBlock == nil { + log.Fatal("Invalid PEM format") + } + + privateKey, err := x509.ParsePKCS1PrivateKey(pemBlock.Bytes) + if err != nil { + log.Fatal(err) + } + + jsonRequest := fmt.Sprintf(`{"@context":["https://www.w3.org/ns/activitystreams","https://%[2]s/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://%[2]s/relay","cc":[],"id":"https://%[2]s/activities/bd0614f5-371d-4bd2-88b3-1ee12f7bf42a","object":"%[1]s","state":"pending","to":["%[1]s"],"type":"Follow"}`, inboxurl, settings.Hostname) + + //var jsonBytes []byte + jsonBytes := []byte(jsonRequest) + payload := bytes.NewReader(jsonBytes) + + prefs := []httpsig.Algorithm{httpsig.RSA_SHA256} + digestAlgorithm := httpsig.DigestSha256 + headers := []string{httpsig.RequestTarget, "date", "host"} + signer, _, err := httpsig.NewSigner(prefs, digestAlgorithm, headers, httpsig.Signature, 0) + if err != nil { + log.Fatal(err.Error()) + } + + req, err := http.NewRequest("POST", inboxurl, payload) + if err != nil { + log.Fatal(err.Error()) + } + if payload != nil { + req.Header.Add("content-type", "application/json") + } + req.Header.Add("date", time.Now().UTC().Format(http.TimeFormat)) + req.Header.Add("host", inboxhostname) + + keyID := "https://" + settings.Hostname + "/relay" + + err = signer.SignRequest(privateKey, keyID, req, jsonBytes) + if err != nil { + log.Fatal(err.Error()) + } + req.Header["Signature"][0] = strings.Replace(req.Header["Signature"][0], "algorithm=\"hs2019\"", "algorithm=\"rsa-sha256\"", 1) + + customTransport := http.DefaultTransport.(*http.Transport).Clone() + customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + client := &http.Client{Timeout: 10 * time.Second, Transport: customTransport} + + // client := &http.Client{Timeout: 10 * time.Second} + res, err := client.Do(req) + if err != nil { + log.Fatal(err) + } + + defer res.Body.Close() +} diff --git a/fedilogue/go.mod b/fedilogue/go.mod new file mode 100644 index 0000000..c219cf1 --- /dev/null +++ b/fedilogue/go.mod @@ -0,0 +1,29 @@ +module git.farhan.codes/farhan/fedilogue/fedilogue + +go 1.23.5 + +require ( + git.farhan.codes/farhan/fedilogue/shared v0.0.0-20250124035748-fa4db6191391 + github.com/go-fed/httpsig v1.1.0 + github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.3 + github.com/jackc/pgx/v4 v4.18.3 + github.com/microcosm-cc/bluemonday v1.0.27 +) + +require ( + github.com/aymerick/douceur v0.2.0 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.14.3 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect + github.com/jackc/puddle v1.3.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect +) diff --git a/fedilogue/go.sum b/fedilogue/go.sum new file mode 100644 index 0000000..4f10f5d --- /dev/null +++ b/fedilogue/go.sum @@ -0,0 +1,187 @@ +git.farhan.codes/farhan/fedilogue/shared v0.0.0-20250124035748-fa4db6191391 h1:/pt4xgnZGxS/pC+CpRQ498gfbOy1E3o16eGKch4AR10= +git.farhan.codes/farhan/fedilogue/shared v0.0.0-20250124035748-fa4db6191391/go.mod h1:KT2OwXD90rPF8qPL/pJVg4WRwq4qvLQcMnoyldckXXw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/fedilogue/instance.go b/fedilogue/instance.go new file mode 100644 index 0000000..2ab2c57 --- /dev/null +++ b/fedilogue/instance.go @@ -0,0 +1,282 @@ +package main + +import ( + "context" + "encoding/json" + "net" + "net/http" + "time" + + "git.farhan.codes/farhan/fedilogue/shared" +) + +var staggeredStartChan chan bool + +func DoTries(o *shared.RunningInstance, req *http.Request) (*http.Response, error) { + var resp *http.Response + var err error + + for tries := 0; tries < 10; tries++ { + resp, err = o.Client.Do(req) + if err != nil { + // URL.Scheme, Host, Path Opaque + logWarn("Failure connecting to "+req.URL.Scheme+"://"+req.URL.Host+req.URL.Path+", attempt ", tries+1, ", sleeping for 5 minutes: ", err) + time.Sleep(time.Minute * 5) + continue + } + break + } + return resp, err +} + +func BuildClient(endpoint string) http.Client { + logDebug("BuildClient for ", endpoint) + // Test: TestBuildClient, TestBuildClientProxy + /* The seemingly unused 'endpoint' variable is for proxying based on endpoint, ie for Tor */ + tr := &http.Transport{ + MaxIdleConns: 2, + IdleConnTimeout: 3600 * time.Second, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + } + client := http.Client{Transport: tr} + + return client +} + +func GetRunner(endpoint string) (shared.RunningInstance, bool) { + // Tests: TestGetRunnerNonExist, TestGetRunnerExists + ri_mutex.Lock() + o, exists := runninginstances[endpoint] + + if exists == false { + o = shared.RunningInstance{} + selectRet := pool.QueryRow(context.Background(), "SELECT banned, alwaysbot FROM instances WHERE endpoint = $1", endpoint) + err := selectRet.Scan(&o.Banned, &o.Alwaysbot) + if err != nil { + logDebug("Did not find instance in database: ", endpoint) + } + if o.Banned == true { + logInfo("Banned instance: ", endpoint) + } else { + logDebug("Building runner for: ", endpoint) + o.Client = BuildClient(endpoint) + o.Status = shared.KEEPALIVE + o.Recentactivities = shared.NewUniqueFifo(10) + o.Recentactors = shared.NewUniqueFifo(10) + } + runninginstances[endpoint] = o + } + ri_mutex.Unlock() + + return o, exists +} + +func UpdateRunner(endpoint string, o shared.RunningInstance) { + // Tests: None necessary + ri_mutex.Lock() + runninginstances[endpoint] = o + ri_mutex.Unlock() +} + +func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningInstance { + /* Checking order + * Mastodon/Pleroma/Misskey + * Um..nothing else yet + */ + logDebug("GetInstanceInfo for ", endpoint) + var nodeinfo shared.NodeInfo + pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json" + + // Checking Mastodon and Pleroma (with .json) + logDebug("GetInstanceInfo: For " + endpoint + " Mastodon/Pleroma URL: " + pleromastodon_nodeinfo_uri) + reqjson, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil) + reqjson.Header.Set("User-Agent", "Tusky") + + pleromastodon_api_resp, err := DoTries(&o, reqjson) + if err != nil { + o.Software = "Unsupported" + return o + } else { + defer pleromastodon_api_resp.Body.Close() + } + + if pleromastodon_api_resp.StatusCode == 200 { + err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo) + if err == nil { + o.Software = nodeinfo.Software.Name + o.Version = nodeinfo.Software.Version + o.LastRun = time.Now().Format(time.RFC3339) + defer pleromastodon_api_resp.Body.Close() + return o + } + } + + // Checking for Misskey (without .json) + misskey_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0" + logDebug("GetInstanceInfo: For " + endpoint + " MissKey URL: " + misskey_nodeinfo_uri) + req, _ := http.NewRequest("GET", misskey_nodeinfo_uri, nil) + req.Header.Set("User-Agent", "Tusky") + + misskey_api_resp, err := DoTries(&o, req) + if err != nil { + o.Software = "Unsupported" + return o + } else { + defer misskey_api_resp.Body.Close() + } + + if misskey_api_resp.StatusCode == 200 { + err = json.NewDecoder(misskey_api_resp.Body).Decode(&nodeinfo) + if err == nil { + o.Software = nodeinfo.Software.Name + o.Version = nodeinfo.Software.Version + o.LastRun = time.Now().Format(time.RFC3339) + defer misskey_api_resp.Body.Close() + return o + } + } + + // Check for Lemmy (With Json) + lemmy_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.1" + logDebug("GetInstanceInfo: For " + endpoint + " Lemmy URL: " + lemmy_nodeinfo_uri) + req, _ = http.NewRequest("GET", lemmy_nodeinfo_uri, nil) + req.Header.Set("User-Agent", "Tusky") + + lemmy_api_resp, err := DoTries(&o, req) + if err != nil { + o.Software = "Unsupported" + return o + } else { + defer lemmy_api_resp.Body.Close() + } + + if lemmy_api_resp.StatusCode == 200 { + err = json.NewDecoder(lemmy_api_resp.Body).Decode(&nodeinfo) + if err == nil { + logDebug("Found a new Lemmy instance: " + endpoint) + o.Software = nodeinfo.Software.Name + o.Version = nodeinfo.Software.Version + o.LastRun = time.Now().Format(time.RFC3339) + defer lemmy_api_resp.Body.Close() + return o + } + } + + // Unsupported Software + o.Software = "Unsupported" + o.Version = "Unknown" + + return o +} + +func LogInstance(endpoint string, o shared.RunningInstance) bool { + logDebug("LogInstance: ", endpoint) + selectRet := pool.QueryRow(context.Background(), "SELECT FROM instances WHERE endpoint = $1", endpoint) + err := selectRet.Scan() + if err == nil { + return true // Endpoint already in database, continuing + } + + _, err = pool.Exec(context.Background(), "INSERT INTO instances (endpoint, state, software) VALUES($1, $2, $3)", endpoint, "", o.Software) + if err != nil { + logWarn("Error inserting ", endpoint+" into `instances`: ", err) + return true + } + + return false +} + +func CheckInstance(newinstance string, callerEndpoint string) { + logDebug("checkInstance: ", newinstance) + if settings.Crawl == true { + // Skip over this if its the same as the endpoint or empty + if newinstance == callerEndpoint || newinstance == "" { + return + } + + var err error + for attempt := 0; attempt > 5; attempt = attempt + 1 { + _, err = net.LookupHost(newinstance) + if err != nil { + logDebug("Unable to resolve "+newinstance+" attempt ", attempt, "/5. Sleeping for 30 seconds") + time.Sleep(time.Second * 30) + continue + } + break + } + if err != nil { + logWarn("Unable to resolve ", newinstance, " after 5 attempts, giving up: ", err) + return + } + + // Skip over this if its the same as the endpoint + if newinstance == callerEndpoint { + return + } + + // Going forward, this might be merged into GetRunner + ri_mutex.Lock() + o, exists := runninginstances[newinstance] + if exists == false || o.Status == shared.KEEPALIVE { + m := shared.RunningInstance{} + m.Client = BuildClient(newinstance) + m.Recentactivities = shared.NewUniqueFifo(10) + m.Recentactors = shared.NewUniqueFifo(10) + runninginstances[newinstance] = m + go StartInstance(newinstance) + } + ri_mutex.Unlock() + } else { + logDebug("Not crawling into ", newinstance) + } + +} + +func staggeredStart() { + for { + _: + <-staggeredStartChan + time.Sleep(500 * time.Millisecond) + } +} + +func StartInstance(endpoint string) { + staggeredStartChan <- true + logInfo("Starting " + endpoint) + + // Check if exists. If so, get the object. If not, create it + o, _ := GetRunner(endpoint) + if o.Banned == true { + logInfo("Ignoring banned instance: ", endpoint) + return // banned instance + } + + o = GetInstanceInfo(endpoint, o) + UpdateRunner(endpoint, o) + LogInstance(endpoint, o) + + if o.Software == "pleroma" { + logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version) + o.CaptureType = "Stream" + UpdateRunner(endpoint, o) + StreamPleroma(endpoint) + } else if o.Software == "mastodon" { + logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version) + o.CaptureType = "Stream" + UpdateRunner(endpoint, o) + StreamMastodon(endpoint, &o) + } else if o.Software == "misskey" { + logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version) + o.CaptureType = "Stream" + UpdateRunner(endpoint, o) + StreamMisskey(endpoint) + } else { + o.Status = 605 + UpdateRunner(endpoint, o) + logConn("Unsupported endpoint " + endpoint) + } +} diff --git a/fedilogue/instance_test.go b/fedilogue/instance_test.go new file mode 100644 index 0000000..9209d27 --- /dev/null +++ b/fedilogue/instance_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "net" + "net/http" + "reflect" + "testing" + "time" + + "git.farhan.codes/farhan/fedilogue/shared" +) + +func TestBuildClient(t *testing.T) { + tr := &http.Transport{ + MaxIdleConns: 2, + IdleConnTimeout: 3600 * time.Second, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + } + want := http.Client{Transport: tr} + have := BuildClient("testdomain.com") + + if reflect.DeepEqual(want, have) { + t.Fatalf("TestBuildClient client different from expected.") + } +} + +func TestBuildClientProxy(t *testing.T) { + // Currently not implemented +} + +func TestGetRunnerNonExist(t *testing.T) { + // Currently not implemented +} + +func TestGetRunnerExists(t *testing.T) { + defer func() { + runninginstances = map[string]shared.RunningInstance{} + }() + + want_o := shared.RunningInstance{} + want_o.Client = BuildClient("some-non-existent-domain.tld") + want_o.Status = shared.KEEPALIVE + want_o.Recentactivities = shared.NewUniqueFifo(10) + want_o.Recentactors = shared.NewUniqueFifo(10) + runninginstances["some-non-existent-domain.tld"] = want_o + + want_exists := true + _, have_exists := GetRunner("some-non-existent-domain.tld") + + if have_exists != want_exists { + t.Fatalf("TestGetRunnerBlank expected %v, got %v", want_exists, have_exists) + } + // if reflect.DeepEqual(want_o, have_o) { + // t.Fatalf("TestGetRunnerExists failed, should have the same value") + // } +} diff --git a/fedilogue/log.go b/fedilogue/log.go new file mode 100644 index 0000000..19eac15 --- /dev/null +++ b/fedilogue/log.go @@ -0,0 +1,58 @@ +package main + +import ( + "log" + "os" +) + +var ( + _logInfo *log.Logger + _logErr *log.Logger + _logWarn *log.Logger + _logConn *log.Logger + _logDebug *log.Logger + _logFatal *log.Logger +) + +func logInit() { + _logInfo = log.New(os.Stdout, "INFO: ", log.Lmsgprefix|log.Ldate|log.Ltime) + _logErr = log.New(os.Stdout, "ERR: ", log.Lmsgprefix|log.Ldate|log.Ltime) + _logWarn = log.New(os.Stdout, "WARN: ", log.Lmsgprefix|log.Ldate|log.Ltime) + _logConn = log.New(os.Stdout, "CONN: ", log.Lmsgprefix|log.Ldate|log.Ltime) + _logDebug = log.New(os.Stdout, "DEBUG: ", log.Lmsgprefix|log.Ldate|log.Ltime) + _logFatal = log.New(os.Stdout, "FATAL: ", log.Lmsgprefix|log.Ldate|log.Ltime) +} + +func logInfo(s ...interface{}) { + if settings.LogLevel >= 1 { + _logInfo.Print(s...) + } +} + +func logErr(s ...interface{}) { + if settings.LogLevel >= 2 { + _logErr.Print(s...) + } +} + +func logWarn(s ...interface{}) { + if settings.LogLevel >= 3 { + _logWarn.Print(s...) + } +} + +func logConn(s ...interface{}) { + if settings.LogLevel >= 4 { + _logConn.Print(s...) + } +} + +func logDebug(s ...interface{}) { + if settings.LogLevel >= 5 { + _logDebug.Print(s...) + } +} + +func logFatal(s ...interface{}) { + _logFatal.Fatal(s...) +} diff --git a/fedilogue/oauth.go b/fedilogue/oauth.go new file mode 100644 index 0000000..641d52e --- /dev/null +++ b/fedilogue/oauth.go @@ -0,0 +1,210 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "io" + "io/ioutil" + "net/http" + "os" + + "git.farhan.codes/farhan/fedilogue/shared" +) + +type OAuth struct { + Access_token string `"json:access_token"` + Created_at int `"json:created_at"` + Expires_in int64 `"json:Expires_in"` + Refresh_token string `"json:refresh_token"` +} + +type authError struct { + msg string +} + +func (e *authError) Error() string { + return e.msg +} + +func register_client(endpoint string, o *shared.RunningInstance) error { + requestBodymap, _ := json.Marshal(map[string]string{ + "client_name": "Tusky", // Hard-coded in for now... + "scopes": "read write follow push", + "redirect_uris": "urn:ietf:wg:oauth:2.0:oob", + }) + requestBodybytes := bytes.NewBuffer(requestBodymap) + + api_base_apps := "https://" + endpoint + "/api/v1/apps" + + resp, err := o.Client.Post(api_base_apps, "application/json", requestBodybytes) + if err != nil { + logErr("Unable to connect to "+api_base_apps+" ", err) + return err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logErr("Unable to read HTTP response: ", err) + o.Client_id = "" + o.Client_secret = "" + return err + } + + bodymap := make(map[string]string) + err = json.Unmarshal(body, &bodymap) + if err != nil { + logErr("Unable to parse response from "+endpoint+": ", err) + o.Client_id = "" + o.Client_secret = "" + return err + } + + client_file := "clients/" + endpoint + + f, err := os.Create("clients/" + endpoint) + if err != nil { + logErr("Unable to create "+client_file+": ", err) + o.Client_id = "" + o.Client_secret = "" + return err + } + defer f.Close() + + _, err = io.WriteString(f, bodymap["client_id"]+"\n") + if err != nil { + logErr("Unable to write client_id line to file "+client_file+": ", err) + o.Client_id = bodymap["client_id"] + o.Client_secret = bodymap["client_secret"] + return nil + } + _, err = io.WriteString(f, bodymap["client_secret"]+"\n") + if err != nil { + logErr("Unable to write client_secret to file "+client_file+": ", err) + o.Client_id = bodymap["client_id"] + o.Client_secret = bodymap["client_secret"] + return nil + } + + o.Client_id = bodymap["client_id"] + o.Client_secret = bodymap["client_secret"] + return nil +} + +func get_client(endpoint string, o *shared.RunningInstance) error { + var err error + client_file := "clients/" + endpoint + _, err = os.Stat(client_file) + if os.IsNotExist(err) == false { // The file exists + f, err := os.Open(client_file) + if err != nil { + logErr("Unable to open " + client_file + ", creating new client") + return err + } + defer f.Close() + + rd := bufio.NewReader(f) + + client_id_bin, _, err := rd.ReadLine() + o.Client_id = string(client_id_bin) + if err != nil { + logErr("Unable to read client_id line of " + client_file + ", building new client") + return err + } + client_secret_bin, _, err := rd.ReadLine() + o.Client_secret = string(client_secret_bin) + if err != nil { + logErr("Unable to read client_secret line of " + client_file + ", building new client") + return err + } + + return nil + } else { + return register_client(endpoint, o) + } + + return nil +} + +func oauth_login(endpoint string, o *shared.RunningInstance, username string, password string) (OAuth, error) { + authMap, err := json.Marshal(map[string]string{ + "username": username, + "password": password, + "redirect_uri": "urn:ietf:wg:oauth:2.0:oob", + "grant_type": "password", + "client_name": "Tusky", + "scope": "read write follow push", + "client_id": o.Client_id, + "client_secret": o.Client_secret, + }) + + if err != nil { + logErr("Unable to create Authentication map for " + endpoint) + return OAuth{}, err + } + + authMapbytes := bytes.NewBuffer(authMap) + + resp, err := http.Post("https://"+endpoint+"/oauth/token", "application/json", authMapbytes) + if err != nil { + logErr("Cannot connect to "+endpoint+": ", err) + return OAuth{}, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logErr("Unable to read response data for "+endpoint+": ", err) + return OAuth{}, err + } + + if resp.StatusCode == 400 { + logErr("Unable to authenticate to " + endpoint) + return OAuth{}, &authError{"Authentication error"} + } + + oauthData := OAuth{} + err = json.Unmarshal(body, &oauthData) + if err != nil { + logErr("Unable to parse json data for "+endpoint+": ", err) + return OAuth{}, err + } + + return oauthData, nil +} + +func oauth_refresh(endpoint string, client_id string, client_secret string, refresh_token string) (OAuth, error) { + authMap, _ := json.Marshal(map[string]string{ + "redirect_uri": "urn:ietf:wg:oauth:2.0:oob", + "grant_type": "refresh_token", + "scope": "read write follow push", + "refresh_token": refresh_token, + "client_id": client_id, + "client_secret": client_secret, + }) + + authMapbytes := bytes.NewBuffer(authMap) + + resp, err := http.Post("https://"+endpoint+"/oauth/token", "application/json", authMapbytes) + if err != nil { + logErr("Unable to connect to "+endpoint+": ", err) + return OAuth{}, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logErr("Unable to read response data for "+endpoint+": ", err) + return OAuth{}, err + } + + oauthData := OAuth{} + err = json.Unmarshal(body, &oauthData) + if err != nil { + logErr("Unable to parse json data for "+endpoint+": ", err) + return oauthData, err + } + + return oauthData, nil +} diff --git a/fedilogue/poll.go b/fedilogue/poll.go new file mode 100644 index 0000000..b7a9c8b --- /dev/null +++ b/fedilogue/poll.go @@ -0,0 +1,164 @@ +package main + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "time" + + "git.farhan.codes/farhan/fedilogue/shared" +) + +type ImageData struct { + Type string `"json:type"` + Url string `"json:url"` +} + +type PublicKeyData struct { + Id string `"json:id"` + Owner string `"json:owner"` + PublicKeyPem string `"json:publicKeyPem"` +} + +type UserInfo struct { + Id string `"json:id"` + Type string `"json:type"` + Following string `"json:following"` + Followers string `"json:followers"` + Inbox string `"json:inbox"` + Outbox string `"json:outbox"` + Featured string `"json:featured"` + PreferredUsername string `"json:preferredUsername"` + PublicKey PublicKeyData `"json:publicKeyPem"` + Name string `"json:name"` + Summary string `"json:summary"` + Url string `"json:Url"` + Icon ImageData `"json:icon"` + Image ImageData `"json:image"` +} + +type PostInfo struct { + Id string `"json:id"` + Type string `"json:type"` + Published string `"json:published"` + Url string `"json:Url"` + Content string `"json:content"` +} + +func PollMastodonPleroma(endpoint string, o *shared.RunningInstance) { + newactivities := make([]shared.ReportActivity, 0) + + min_id := "" + + parsing_error := 0 + use_auth := false + + var last_refresh int64 + var client_id string + var client_secret string + var oauthData OAuth + + for { + ri_mutex.Lock() + m := runninginstances[endpoint] + ri_mutex.Unlock() + + api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&since_id=" + min_id + req, err := http.NewRequest("GET", api_timeline, nil) + req.Header.Set("User-Agent", "Tusky") + if err != nil { + logFatal("Unable to create new request for "+endpoint+": ", err) + return + } + + if use_auth == true { + if time.Now().Unix() > last_refresh+oauthData.Expires_in { + oauthData, err = oauth_refresh(endpoint, client_id, client_secret, oauthData.Refresh_token) + if err != nil { + logWarn("Unable to refresh oauth token for "+endpoint+": ", err) + return + } + last_refresh = time.Now().Unix() + } + req.Header.Add("Authorization", oauthData.Access_token) + } + + m.LastRun = time.Now().Format(time.RFC3339) + resp, err := DoTries(o, req) + if err != nil { + m.Status = shared.CLIENT_ISSUE + ri_mutex.Lock() + runninginstances[endpoint] = m + ri_mutex.Unlock() + logWarn("Giving up on "+endpoint+": ", err.Error()) + return + } + + if resp.StatusCode == shared.TOOMANYREQUESTS { // Short Delay, 30 seconds + logWarn("Delaying "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay") + _, _ = ioutil.ReadAll(resp.Body) + resp.Body.Close() // Release as soon as done + m.Status = resp.StatusCode + ri_mutex.Lock() + runninginstances[endpoint] = m + ri_mutex.Unlock() + time.Sleep(time.Second * 30) + continue + } else if resp.StatusCode == shared.INTERNAL_ERROR { // Longer delay, 1 hour + logWarn("Suspending "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay") + _, _ = ioutil.ReadAll(resp.Body) + resp.Body.Close() // Release as soon as done + m.Status = 765 + ri_mutex.Lock() + runninginstances[endpoint] = m + ri_mutex.Unlock() + time.Sleep(time.Second * 3600) + continue + } else if resp.StatusCode != 200 { // Crash + logErr("Terminating "+endpoint+", gave status ", resp.StatusCode) + _, _ = ioutil.ReadAll(resp.Body) + resp.Body.Close() // Release as soon as done + m.Status = resp.StatusCode + ri_mutex.Lock() + runninginstances[endpoint] = m + ri_mutex.Unlock() + return + } + + err = json.NewDecoder(resp.Body).Decode(&newactivities) + resp.Body.Close() // Release as soon as done + if err != nil { + if parsing_error > 5 { + m.Status = shared.BAD_RESPONSE + ri_mutex.Lock() + runninginstances[endpoint] = m + ri_mutex.Unlock() + logErr("Giving up on " + endpoint + " after 5 unmarshal errors.") + return + } + parsing_error = parsing_error + 1 + time.Sleep(time.Second * 30) + } + + m.Status = shared.RUNNING + ri_mutex.Lock() + runninginstances[endpoint] = m + ri_mutex.Unlock() + + for _, newactivity := range newactivities { + go check_activity(newactivity.Uri) + matchset := re.FindStringSubmatch(newactivity.Uri) + if matchset != nil { + newinstance := matchset[1] + + // Check min_id + if newactivity.Id > min_id { + min_id = newactivity.Id + } + + go CheckInstance(newinstance, endpoint) + } + } + time.Sleep(time.Second * 10) + } +} diff --git a/fedilogue/retrieve.go b/fedilogue/retrieve.go new file mode 100644 index 0000000..5059414 --- /dev/null +++ b/fedilogue/retrieve.go @@ -0,0 +1,287 @@ +package main + +import ( + "context" + "encoding/json" + "html" + "io/ioutil" + "net/http" + "regexp" + "strings" + "time" + + "github.com/microcosm-cc/bluemonday" +) + +var p *bluemonday.Policy +var spaceReg *regexp.Regexp +var removeHTMLReg *regexp.Regexp +var re *regexp.Regexp +var matchurl *regexp.Regexp + +type ImageType struct { + Url string `json:"url"` +} + +type PublicKeyType struct { + PublicKeyPem string `json:"publicKeyPem"` +} + +type ActorJson struct { + id int + Uri string `json:"id"` + Type string `json:"type"` + Inbox string `json:"inbox"` + Outbox string `json:"outbox"` + Followers string `json:"followers"` + Following string `json:"following"` + Url string `json:"url"` + PreferredUsername string `json:"preferredUsername"` + Name string `json:"name"` + Summary string `json:"summary"` + Icon ImageType `json:"icon"` + Image ImageType `json:"image"` + PublicKey PublicKeyType `json:"publicKey"` + + bot bool + instance string +} + +type TagType struct { + Type string `json:"type"` + Name string `json:"name"` +} + +type PostJson struct { + id int + Uri string `json:"id"` + InReplyTo string `json:"inReplyTo"` + + normalized string + receivedAt time.Time `json:"created_at"` + + Content string `json:"content"` + Conversation string `json:"conversation"` + Published time.Time `json:"published"` + Summary string `json:"summary"` + Tag []TagType `json:"tag"` + To []string `json:"to"` + Type string `json:"type"` + + Actor string `json:"actor"` + AttributedTo string `json:"attributedTo"` + + bot bool + instance string +} + +func check_activity(uri string) { + logDebug("Retrieving: " + uri) + var activityjson PostJson + + // Ignore invalid URIs + endslash := strings.Index(uri[8:], "/") + if endslash == -1 { + return + } + activityjson.instance = uri[8 : endslash+8] + + o, _ := GetRunner(activityjson.instance) + if o.Banned == true { + logDebug("Ignoring banned instance: ", uri) + return // Banned instance + } + + // Check if there were any recent requests on this + o.Recentactivities.Mu.Lock() + i, _ := o.Recentactivities.Contains(uri) + if i != -1 { + logDebug("Ignoring cached recent request: ", uri) + o.Recentactivities.Mu.Unlock() + return + } + + o.Recentactivities.Add(uri, "") // Added blank entry + o.Recentactivities.Mu.Unlock() + var jsondocument string + + selectRet := pool.QueryRow(context.Background(), "SELECT FROM activities WHERE document->>'id' = $1", uri) + err := selectRet.Scan() + if err == nil { + logDebug("Already in database, ignoring: ", uri) + return + } + + req, _ := http.NewRequest("GET", uri, nil) + req.Header.Set("User-Agent", "Tusky") + req.Header.Add("Accept", "application/ld+json") + + resp, err := DoTries(&o, req) + if err != nil { + logDebug("Gave up after multiple tries: ", uri) + return + } + + if resp.StatusCode != 200 { + logDebug("Non-200 response code for ", uri, " was ", resp.StatusCode) + resp.Body.Close() + return + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logDebug("Failed to read the reply: ", uri) + return + } + resp.Body.Close() + jsondocument = string(body) + err = json.Unmarshal(body, &activityjson) + if err != nil { + logDebug("Failed to Unmarshal, err: ", err, " uri: ", uri) + return + } + + if activityjson.InReplyTo != "" && activityjson.InReplyTo != uri { + if activityjson.InReplyTo != uri { + go check_activity(activityjson.InReplyTo) + } + } + + // If AttributedTo is blank, this is likely an authentication failure + // For now, skip it... + if activityjson.AttributedTo == "" { + logDebug("AttributedTo field is blank, dropping for ", uri) + return + } + + // This must be done BEFORE the `INSERT INTO activities'` below + actorjson := check_actor(activityjson.AttributedTo) + if actorjson == nil { + logDebug("Failed to add actor, dropping post: ", uri) + return + } + if actorjson.bot || o.Alwaysbot { + activityjson.bot = true + } + + activityjson.normalized = removeHTMLReg.ReplaceAllString(activityjson.Content, " ") + activityjson.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(activityjson.normalized))) + activityjson.normalized = matchurl.ReplaceAllString(activityjson.normalized, "") + activityjson.normalized = spaceReg.ReplaceAllString(activityjson.normalized, " ") + + var hashtags []string + for _, tag := range activityjson.Tag { + if tag.Type == "Hashtag" { + hashtags = append(hashtags, strings.ToLower(tag.Name)) + } + } + _, err = pool.Exec(context.Background(), "INSERT INTO activities (document, normalized, instance, hashtags, bot) VALUES($1, $2, $3, $4, $5)", jsondocument, activityjson.normalized, activityjson.instance, hashtags, activityjson.bot) + if err != nil { + logWarn("Error inserting ", uri, " into `activities`: ", err) + return + } + + for _, to := range activityjson.To { + if to != "https://www.w3.org/ns/activitystreams#Public" && to != "" { + if strings.HasSuffix(to, "/followers") { + // This check is very much a bad solution, may consider removing the entire for-loop + continue + } + go check_actor(to) + } + } + +} + +/* Test: TestCheck_actor */ +func check_actor(uri string) *ActorJson { + actorjson := &ActorJson{} + + if len(uri) <= 7 { + return nil // Bad actor + } + + endslash := strings.Index(uri[8:], "/") + if endslash == -1 { + return nil // Bad actor + } + actorjson.instance = uri[8 : endslash+8] + + // Check if there were any recent requests on this + o, _ := GetRunner(actorjson.instance) + if o.Banned { + logDebug("Banned actor: ", uri) + return nil // Banned actor + } + o.Recentactors.Mu.Lock() + i, cachedactorjson := o.Recentactors.Contains(uri) + if i != -1 { + o.Recentactors.Mu.Unlock() + cachedactorjson := cachedactorjson.(*ActorJson) + return cachedactorjson + } + o.Recentactors.Mu.Unlock() + + selectRet := pool.QueryRow(context.Background(), "SELECT document FROM actors WHERE document->>'id' = $1", uri) + err := selectRet.Scan(&actorjson) + if err == nil { + return actorjson // Actor already in database, good! + } + + req, _ := http.NewRequest("GET", uri, nil) + req.Header.Set("User-Agent", "Tusky") + req.Header.Add("Accept", "application/ld+json") + + var resp *http.Response + tries := 0 + for { + resp, err = o.Client.Do(req) + if err != nil { + if tries > 10 { + logErr("Unable to connect to " + uri + " attempt 10/10, giving up.") + return nil // Unable to connect to host after 10 attempts + } + logWarn("Unable to connect to "+uri+", attempt ", tries+1, "+/10 sleeping for 30 seconds.") + time.Sleep(time.Second * 30) + tries = tries + 1 + continue + } + break + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logWarn("Unable to read body from ", uri) + return nil // Unable to read body of message + } + resp.Body.Close() + + jsondocument := string(body) + + err = json.Unmarshal(body, &actorjson) + if err != nil { + logWarn("Unable to unmarshal body from ", uri) + return nil // Unable to unmarshal body of message + } + o.Recentactors.Mu.Lock() + o.Recentactors.Add(uri, actorjson) + o.Recentactors.Mu.Unlock() + + var bot bool + if actorjson.Type == "Service" { + actorjson.bot = true + } else { + actorjson.bot = o.Alwaysbot // default on host's classification + } + + _, err = pool.Exec(context.Background(), "INSERT INTO actors (document, instance, bot) VALUES($1, $2, $3)", jsondocument, actorjson.instance, bot) + if err != nil { + logWarn("Error inserting ", uri, " into `actors`: ", err) + return nil // Unable to insert actor + } + + o.Recentactors.Mu.Lock() + o.Recentactors.Add(uri, actorjson) + o.Recentactors.Mu.Unlock() + return actorjson // Successful +} diff --git a/fedilogue/retrieve_test.go b/fedilogue/retrieve_test.go new file mode 100644 index 0000000..2681715 --- /dev/null +++ b/fedilogue/retrieve_test.go @@ -0,0 +1,9 @@ +package main + +import ( + "testing" +) + +func TestCheck_actor(t *testing.T) { + // Currently not implemented +} diff --git a/fedilogue/stream.go b/fedilogue/stream.go new file mode 100644 index 0000000..b799353 --- /dev/null +++ b/fedilogue/stream.go @@ -0,0 +1,294 @@ +package main + +import ( + "bufio" + "encoding/json" + + "net/http" + "net/url" + "strings" + "time" + + "github.com/google/uuid" + "github.com/gorilla/websocket" + "git.farhan.codes/farhan/fedilogue/shared" +) + +type MisskeyReply struct { + Type string `json:"type"` + Body MisskeyReplyBody `json:"body"` +} + +type MisskeyReplyBody struct { + Channel string `json:"channel"` + Type string `json:"type"` + Body MisskeyNoteBody `json:"body"` +} + +type MisskeyNoteBody struct { + Uri string `json:"uri"` // Remote Note + Id string `json:"id"` // Local note +} + +type MisskeyRequest struct { + Type string `json:"type"` + Body MisskeyRequestBody `json:"body"` +} + +type MisskeyRequestBody struct { + Channel string `json:"channel"` + Id string `json:"id"` + Params MisskeyRequestParams +} + +type MisskeyRequestParams struct { +} + +func StreamMisskey(endpoint string) { + logDebug("Stream Misskey for ", endpoint) + u := url.URL{ + Scheme: "wss", + Host: endpoint, + Path: "/streaming", + } + + var misskeyrequest MisskeyRequest + misskeyrequest.Type = "connect" + misskeyrequest.Body.Channel = "globalTimeline" + misskeyrequest.Body.Id = uuid.New().String() + + for { + misskeyrequeststream, err := json.Marshal(misskeyrequest) + if err != nil { + panic(err) + } + + for { + // Create a new WebSocket connection. + ws, _, err := websocket.DefaultDialer.Dial(u.String(), nil) + if err != nil { + logErr("Error dialing misskey webSocket to ", endpoint, " :", err) + return + } + logDebug("Misskey websocket connection created: ", endpoint) + + ri_mutex.Lock() + m := runninginstances[endpoint] + m.Status = shared.RUNNING + m.LastRun = "Streaming" + runninginstances[endpoint] = m + ri_mutex.Unlock() + + // Send a message to the server. + err = ws.WriteMessage(websocket.TextMessage, misskeyrequeststream) + if err != nil { + logErr("Error sending misskey channel subscription: ", endpoint) + return + } + logDebug("Successfully sent misskey channel subscription: ", endpoint) + + // Read a message from the server. + for { + logDebug("Starting Misskey Stream loop for ", endpoint) + _, message, err := ws.ReadMessage() + if err != nil { + logErr("Misskey stream broken: ", endpoint) + return + } + + // Print the message to the console. + logDebug("Ending Misskey Stream loop for ", endpoint) + var misskeyreply MisskeyReply + err = json.Unmarshal(message, &misskeyreply) + if err != nil { + logErr("Unable to parse data from "+endpoint+", but still connected, err: ", err) + break + } + // newactivity := misskeyreply.Body.Body + + var newactivity string + + if misskeyreply.Body.Body.Uri != "" { // Remote Message + newactivity = misskeyreply.Body.Body.Uri + matchset := re.FindStringSubmatch(newactivity) + if matchset != nil { + newinstance := matchset[1] + logDebug("Checking new instance from Misskey Stream: ", newinstance) + go CheckInstance(newinstance, endpoint) + } + } else { // Local Message + newactivity = "https://" + endpoint + "/notes/" + misskeyreply.Body.Body.Id + } + + logDebug("Misskey new URI ", newactivity, " from instance: ", endpoint) + + go check_activity(newactivity) + } + // Close the WebSocket connection. + ws.Close() + time.Sleep(time.Minute * 10) + } + } +} + +func StreamPleroma(endpoint string) { + wss_url := "wss://" + endpoint + "/api/v1/streaming/?stream=public" + var pleromaHeader shared.PleromaStreamHeader + var newactivity shared.ReportActivity + var err error + + for { + var tries int + var ws *websocket.Conn + for tries = 0; tries < 10; tries++ { + ws, _, err = websocket.DefaultDialer.Dial(wss_url, nil) + if err != nil { + continue + } + break + } + + if tries == 10 { + logWarn("Unable to connect to " + endpoint + " after 10 tries, exiting") + return + } + + ri_mutex.Lock() + m := runninginstances[endpoint] + m.Status = shared.RUNNING + m.LastRun = "Streaming" + runninginstances[endpoint] = m + ri_mutex.Unlock() + + for { + logDebug("Starting Pleroma Stream loop for ", endpoint) + _, p, err := ws.ReadMessage() + if err != nil { + logErr("Unable to read message from Pleroma stream: ", endpoint, " Err: ", err) + break + } + + err = json.Unmarshal(p, &pleromaHeader) + if err != nil { + logErr("Unable to parse data from "+endpoint+", but still connected, err: ", err) + break + } + + switch pleromaHeader.Event { + case "update": + err = json.Unmarshal([]byte(pleromaHeader.Payload), &newactivity) + if err != nil { + logErr("Unable to parse data from " + endpoint + ", but still connected.") + break + } + go check_activity(newactivity.Uri) + matchset := re.FindStringSubmatch(newactivity.Uri) + if matchset != nil { + newinstance := matchset[1] + logDebug("Checking new instance from Pleroma Stream: ", newinstance) + go CheckInstance(newinstance, endpoint) + } + default: + logDebug("Unimplemented pleroma stream activity: ", pleromaHeader.Event) + continue + } + logDebug("Ending Pleroma stream loop for ", endpoint) + } + ws.Close() + // time.Sleep(time.Minute * 10) + } +} + +func StreamMastodon(endpoint string, o *shared.RunningInstance) { + logDebug("Stream Mastodon for ", endpoint) + stream_client := BuildClient(endpoint) + + var retry bool + + api_timeline := "https://" + endpoint + "/api/v1/streaming/public" + + for { + logDebug("StreamMastodon: " + endpoint + " URL: " + api_timeline) + req, err := http.NewRequest("GET", api_timeline, nil) + req.Header.Set("User-Agent", "Tusky") + if err != nil { + logFatal("Unable to create new request for " + endpoint + ", exiting.") + return + } + + var resp *http.Response + + for tries := 0; tries < 10; tries++ { + resp, err = stream_client.Do(req) + if err != nil { + time.Sleep(time.Minute * 5) + logWarn("Failure connecting to "+req.URL.Scheme+"://"+req.URL.Host+req.URL.Path+", attempt ", tries+1, ", sleeping for 5 minutes, ", err) + continue + } + break + } + if err != nil { + logErr("Unable to stream "+api_timeline+": ", err) + return + } + defer resp.Body.Close() + + ri_mutex.Lock() + m := runninginstances[endpoint] + m.Status = shared.RUNNING + m.LastRun = "Streaming" + runninginstances[endpoint] = m + ri_mutex.Unlock() + + s := bufio.NewScanner(resp.Body) + var name string + for s.Scan() { + logDebug("Starting Mastodon stream loop for ", endpoint) + line := s.Text() + token := strings.SplitN(line, ":", 2) + var newactivity shared.ReportActivity + if len(token) != 2 { + continue + } + + switch strings.TrimSpace(token[0]) { + case "event": + name = strings.TrimSpace(token[1]) + continue + case "data": + switch name { + case "update": + if len(token) >= 2 && len(token[1]) >= 2 { + continue + } + jsondata := token[1][1:] + err := json.Unmarshal([]byte(jsondata), &newactivity) + if err != nil { + logDebug("Unable to parse data from " + endpoint + ", but still connected.") + continue + } + retry = true + default: + continue + } + default: + continue + } + + go check_activity(newactivity.Uri) + + matchset := re.FindStringSubmatch(newactivity.Uri) + if matchset != nil { + newinstance := matchset[1] + logDebug("Checking new instance from Mastodon Stream: ", newinstance) + go CheckInstance(newinstance, endpoint) + } + logDebug("Ending Mastodon stream loop for ", endpoint) + } + if retry == true { + time.Sleep(time.Minute * 10) + } else { + break + } + } +} diff --git a/fedilogue/testhelper.go b/fedilogue/testhelper.go new file mode 100644 index 0000000..9baca3a --- /dev/null +++ b/fedilogue/testhelper.go @@ -0,0 +1,15 @@ +package main + +import ( + "reflect" + "testing" +) + +// AssertEqual checks if values are equal +func AssertEqual(t *testing.T, a interface{}, b interface{}) { + if a == b { + return + } + // debug.PrintStack() + t.Errorf("Received %v (type %v), expected %v (type %v)", a, reflect.TypeOf(a), b, reflect.TypeOf(b)) +} diff --git a/fedilogue/web.go b/fedilogue/web.go new file mode 100644 index 0000000..cf2d028 --- /dev/null +++ b/fedilogue/web.go @@ -0,0 +1,311 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "strings" +) + +// CreateObject - Used by post web receiver +type CreateObject struct { + ID string `json:"id"` + Actor string `json:"actor"` + Cc []string `json:"cc"` + Content string `json:"content"` + To []string `json:"to"` + Type string `json:"type"` +} + +// CreateObject - Used by post web receiver +type AnnounceObject struct { + ID string `json:"id"` + Actor string `json:"actor"` + To []string `json:"to"` + Type string `json:"type"` + Object string `json:"object"` +} + +// RelayBase - The base object used by web receiver +type RelayBase struct { + Actor string `json:"actor"` + Cc []string `json:"cc"` + Object json.RawMessage `json:"Object"` + ID string `json:"id"` + Published string `json:"published"` + To []string `json:"to"` + Type string `json:"type"` +} + +func hostmeta(w http.ResponseWriter, r *http.Request) { + fmt.Println("PATH --> ", r.URL.Path) + host := r.Host + xml := "" + w.Header().Set("Content-Type", "application/xrd+xml") + fmt.Fprintf(w, xml) +} + +func webfinger(w http.ResponseWriter, r *http.Request) { + fmt.Println("PATH --> ", r.URL.Path) + host := r.Host + + webfingermap := make(map[string]interface{}) + webfingermap["subject"] = "acct:fedilogue@" + host + webfingermap["aliases"] = []string{"https://" + host + "/users/fedilogue"} + link0 := make(map[string]string) + link0["rel"] = "http://webfinger.net/rel/profile-page" + link0["type"] = "text/html" + link0["href"] = "https://" + host + "/users/fedilogue" + link1 := make(map[string]string) + link1["rel"] = "self" + link1["type"] = "application/activity+json" + link1["href"] = "https://" + host + "/users/fedilogue" + link2 := make(map[string]string) + link2["rel"] = "self" + link2["type"] = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" + link2["href"] = "https://" + host + "/users/fedilogue" + link3 := make(map[string]string) + link3["rel"] = "http://ostatus.org/schema/1.0/subscribe" + link3["template"] = "https://" + host + "/ostatus_subscribe?acct={uri}" + + links := []map[string]string{link0, link1, link2, link3} + webfingermap["links"] = links + + webfingerbin, err := json.Marshal(webfingermap) + if err != nil { + fmt.Println(err.Error()) + return + } + + webfingerstr := string(webfingerbin) + + query := r.URL.Query() + resourceRaw, exists := query["resource"] + + if exists { + resource := resourceRaw[0] + if resource != "acct:fedilogue@"+host { + fmt.Println("Writes properly but wrong acct") + w.Header().Set("Content-Type", "application/json; charset=utf-8") + fmt.Fprintf(w, webfingerstr) + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + fmt.Fprintf(w, webfingerstr) + } else { + fmt.Println(query) + w.WriteHeader(http.StatusNotFound) + return + } +} + +func inboxHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + fmt.Println(err) + return + } + defer r.Body.Close() + + var findtype RelayBase + err = json.Unmarshal(body, &findtype) + if err != nil { + logWarn("Unable to unmarshal here, exiting...") + return + } + + switch findtype.Type { + case "Create": + var createobject CreateObject + + err = json.Unmarshal(body, &createobject) + if err != nil { + return + } + + go check_activity(createobject.ID) + slashend := strings.Index(createobject.ID[8:], "/") + newinstance := createobject.ID[8 : 8+slashend] + + go CheckInstance(newinstance, "") + + case "Update": + case "Reject": + case "Add": + case "Remove": + case "Follow": + case "Accept": + case "Like": + case "Announce": + var announceobject AnnounceObject + err = json.Unmarshal(body, &announceobject) + if err != nil { + fmt.Println(err.Error()) + return + } + check_activity(announceobject.Object) + + matchset := re.FindStringSubmatch(announceobject.Object) + if matchset != nil { + newinstance := matchset[1] + go CheckInstance(newinstance, "") + } + case "Delete": + case "Undo": + default: + logWarn("Unknown ActivityPub request:", findtype.Type) + } + + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, "{}") + } +} + +func internalFetch(w http.ResponseWriter, r *http.Request) { + + publicKeyBytes, err := ioutil.ReadFile("keys/public.pem") + publicKeyString := string(publicKeyBytes) + publicKeyString = strings.Replace(publicKeyString, "\n", "\\n", -11) + if err != nil { + log.Fatal(err) + } + + staticjson := fmt.Sprintf(`{"@context":["https://www.w3.org/ns/activitystreams","https://%[1]s/schemas/litepub-0.1.jsonld",{"@language":"und"}],"endpoints":{"oauthAuthorizationEndpoint":"https://%[1]s/oauth/authorize","oauthRegistrationEndpoint":"https://%[1]s/api/v1/apps","oauthTokenEndpoint":"https://%[1]s/oauth/token","sharedInbox":"https://%[1]s/inbox","uploadMedia":"https://%[1]s/api/ap/upload_media"},"followers":"https://%[1]s/internal/fetch/followers","following":"https://%[1]s/internal/fetch/following","id":"https://%[1]s/internal/fetch","inbox":"https://%[1]s/internal/fetch/inbox","invisible":true,"manuallyApprovesFollowers":false,"name":"Pleroma","preferredUsername":"fedilogue","publicKey":{"id":"https://%[1]s/internal/fetch#main-key","owner":"https://%[1]s/internal/fetch","publicKeyPem":"%[2]s"},"summary":"Fedilogue Key or something.","type":"Application","url":"https://%[1]s/internal/fetch"}`, r.Host, publicKeyString) + + fmt.Fprintf(w, staticjson) + +} + +func relay(w http.ResponseWriter, r *http.Request) { + + fmt.Println("Someone out there requested /relay") + + publicKeyBytes, err := ioutil.ReadFile("keys/public.pem") + publicKeyString := string(publicKeyBytes) + publicKeyString = strings.Replace(publicKeyString, "\n", "\\n", -11) + if err != nil { + log.Fatal(err) + } + + staticjson := fmt.Sprintf(`{"@context":["https://www.w3.org/ns/activitystreams","https://%[1]s/schemas/litepub-0.1.jsonld",{"@language":"und"}],"alsoKnownAs":[],"attachment":[],"capabilities":{},"discoverable":false,"endpoints":{"oauthAuthorizationEndpoint":"https://%[1]s/oauth/authorize","oauthRegistrationEndpoint":"https://%[1]s/api/v1/apps","oauthTokenEndpoint":"https://%[1]s/oauth/token","sharedInbox":"https://%[1]s/inbox","uploadMedia":"https://%[1]s/api/ap/upload_media"},"featured":"https://%[1]s/relay/collections/featured","followers":"https://%[1]s/relay/followers","following":"https://%[1]s/relay/following","id":"https://%[1]s/relay","inbox":"https://%[1]s/relay/inbox","manuallyApprovesFollowers":false,"name":null,"outbox":"https://%[1]s/relay/outbox","preferredUsername":"relay","publicKey":{"id":"https://%[1]s/relay#main-key","owner":"https://%[1]s/relay","publicKeyPem":"%[2]s"},"summary":"","tag":[],"type":"Person","url":"https://%[1]s/relay"}`, r.Host, publicKeyString) + + fmt.Fprintf(w, staticjson) + +} + +func usersFedilogueFollowers(w http.ResponseWriter, r *http.Request) { + fmt.Println("PATH --> ", r.URL.Path) + host := r.Host + contextlist := map[string]string{"@language": "und"} + + context := []interface{}{"https://www.w3.org/ns/activitystreams", "https://" + host + "/schemas/litepub-0.1.jsonld", contextlist} + followersmap := make(map[string]interface{}) + followersmap["@context"] = context + + staticjson := "{\"@context\":[\"https://www.w3.org/ns/activitystreams\",\"https://" + host + "/schemas/litepub-0.1.jsonld\",{\"@language\":\"und\"}],\"first\":{\"id\":\"https://" + host + "/users/fedilogue/followers?page=1\",\"next\":\"https://" + host + "/users/fedilogue/followers?page=2\",\"orderedItems\":[\"https://mastodon.host/users/federationbot\"],\"partOf\":\"https://" + host + "/users/fedilogue/followers\",\"totalItems\":1,\"type\":\"OrderedCollectionPage\"},\"id\":\"https://" + host + "/users/fedilogue/followers\",\"totalItems\":1,\"type\":\"OrderedCollection\"}" + + w.Header().Set("Content-Type", "application/activity+json; charset=utf-8") + fmt.Fprintf(w, staticjson) +} + +func usersFedilogueFollowing(w http.ResponseWriter, r *http.Request) { + host := r.Host + fmt.Println("PATH --> ", r.URL.Path) + staticjson := "{\"@context\": [\"https://www.w3.org/ns/activitystreams\", \"https://" + host + "/schemas/litepub-0.1.jsonld\", {\"@language\": \"und\"}], \"first\": {\"id\": \"https://" + host + "/users/fedilogue/following?page=1\", \"orderedItems\": [], \"partOf\": \"https://" + host + "/users/fedilogue/following\", \"totalItems\": 0, \"type\": \"OrderedCollectionPage\"}, \"id\": \"https://" + host + "/users/fedilogue/following\", \"totalItems\": 0, \"type\": \"OrderedCollection\"}" + w.Header().Set("Content-Type", "application/activity+json; charset=utf-8") + fmt.Fprintf(w, staticjson) +} + +func usersFedilogue(w http.ResponseWriter, r *http.Request) { + fmt.Println("PATH --> ", r.URL.Path) + host := r.Host + + fmt.Println(r.Host) + fmt.Println(r.Header["Accept"]) + + publickeybin, err := ioutil.ReadFile("keys/public.pem") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + publickeypemstr := string(publickeybin) + + publicKey := map[string]string{"id": "https://" + host + "/users/fedilogue#main-key", "owner": "https://" + host + "/users/fedilogue", "publicKeyPem": publickeypemstr} + + capabilities := map[string]bool{} + tag := []string{} + contextlist := map[string]string{"@language": "und"} + attachment := []string{} + endpoints := map[string]string{"oauthAuthorizationEndpoint": "https://" + host + "/oauth/authorize", "oauthRegistrationEndpoint": "https://" + host + "/api/v1/apps", "oauthTokenEndpoint": "https://" + host + "/oauth/token", "sharedInbox": "https://" + host + "/inbox", "uploadMedia": "https://" + host + "/api/ap/upload_media"} + + context := []interface{}{"https://www.w3.org/ns/activitystreams", "https://" + host + "/schemas/litepub-0.1.jsonld", contextlist} + actorjsonmap := make(map[string]interface{}) + actorjsonmap["@context"] = context + actorjsonmap["attachment"] = attachment + actorjsonmap["capabilities"] = capabilities + actorjsonmap["discoverable"] = false + actorjsonmap["endpoints"] = endpoints + actorjsonmap["followers"] = "https://" + host + "/users/fedilogue/followers" + actorjsonmap["following"] = "https://" + host + "/users/fedilogue/following" + actorjsonmap["id"] = "https://" + host + "/users/fedilogue" + actorjsonmap["inbox"] = "https://" + host + "/users/fedilogue/inbox" + actorjsonmap["manuallyApprovesFollowers"] = false + actorjsonmap["name"] = "Fedilogue Mass Follower" + actorjsonmap["outbox"] = "https://" + host + "/users/fedilogue/outbox" + actorjsonmap["preferredUsername"] = "fedilogue" + actorjsonmap["publicKey"] = publicKey + actorjsonmap["summary"] = "" + actorjsonmap["tag"] = tag + actorjsonmap["type"] = "Application" + actorjsonmap["uri"] = "https://" + host + "/users/fedilogue" + + actorjsonbin, err := json.Marshal(actorjsonmap) + if err != nil { + fmt.Println(err.Error()) + return + } + + actorjsonstr := string(actorjsonbin) + w.Header().Set("Content-Type", "application/activity+json; charset=utf-8") + fmt.Fprintf(w, actorjsonstr) +} + +func errorHandler(w http.ResponseWriter, r *http.Request) { + for name, headers := range r.Header { + fmt.Println("ROW: ", name, " ", headers) + for _, h := range headers { + fmt.Fprintf(w, "%v: %v\n", name, h) + } + } + + reqBody, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("POST DATA: %s\n", reqBody) + +} + +func webmain() { + + webreceiver := http.NewServeMux() + + webreceiver.HandleFunc("/.well-known/webfinger", webfinger) + webreceiver.HandleFunc("/.well-known/host-meta", hostmeta) + webreceiver.HandleFunc("/inbox", inboxHandler()) + webreceiver.HandleFunc("/internal/fetch", internalFetch) + webreceiver.HandleFunc("/relay", relay) + webreceiver.HandleFunc("/users/fedilogue", usersFedilogue) + webreceiver.HandleFunc("/users/fedilogue/followers", usersFedilogueFollowers) + webreceiver.HandleFunc("/users/fedilogue/following", usersFedilogueFollowing) + webreceiver.HandleFunc("/", errorHandler) + log.Print("Starting HTTP inbox on port 127.0.0.1:8042") + log.Fatal(http.ListenAndServe("127.0.0.1:8042", webreceiver)) +}