Woops!

fedilogue/config.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+)
+
+// Settings - Configuration file structure
+type Settings struct {
+	Crawl            bool
+	LogLevel         int
+	Hostname         string
+}
+
+var settings Settings
+
+/* Test: TestStringexists */
+func stringexists(needle string, haystack []string) bool {
+	for _, check := range haystack {
+		if check == needle {
+			return true
+		}
+	}
+	return false
+}
+
+func getSettings() {
+	flag.BoolVar(&settings.Crawl, "c", true, "Crawl mode (default is yes)")
+	flag.StringVar(&settings.Hostname, "h", "myhostname", "Set your hostname")
+	flag.IntVar(&settings.LogLevel, "l", 1, "Logging Level:\n  0) No logs\n  1) Reports every 30 seconds\n  2) Errors\n  3) Warnings\n  4) New Connections\n  5) Debugging\n")
+
+	fmt.Println("Crawl: ", settings.Crawl)
+
+	flag.Parse()
+	fmt.Println(settings)
+}

fedilogue/config_test.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestStringexists(t *testing.T) {
+	var empty_strings = []string {}
+	var three_strings = []string {"first", "second", "third"}
+
+	AssertEqual(t, stringexists("amything", empty_strings), false)
+	AssertEqual(t, stringexists("second", three_strings), true)
+	AssertEqual(t, stringexists("fourth", three_strings), false)
+}

fedilogue/ctl.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+)
+
+func cmdFollow(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer r.Body.Close()
+
+	var followtarget interface{}
+	err = json.Unmarshal(body, &followtarget)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	myMap := followtarget.(map[string]interface{})
+	followInbox(myMap["follow"].(string))
+
+	fmt.Fprintf(w, "{}")
+}
+
+func cmdAdd(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer r.Body.Close()
+
+	var addtarget interface{}
+	err = json.Unmarshal(body, &addtarget)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	myMap := addtarget.(map[string]interface{})
+	go StartInstance(myMap["host"].(string))
+
+	fmt.Fprintf(w, "{}")
+}
+
+func cmdStatus(w http.ResponseWriter, r *http.Request) {
+	ri_mutex.Lock()
+	bytes, err := json.Marshal(runninginstances)
+	if err != nil {
+		fmt.Println("Error happened", err)
+	}
+	ri_mutex.Unlock()
+	fmt.Fprintf(w, (string(bytes)))
+}
+
+func startctl() {
+	ctlweb := http.NewServeMux()
+	ctlweb.HandleFunc("/follow", cmdFollow)
+	ctlweb.HandleFunc("/status", cmdStatus)
+	ctlweb.HandleFunc("/add", cmdAdd)
+	log.Print("Starting HTTP inbox on port 127.0.0.1:5555")
+	log.Fatal(http.ListenAndServe("127.0.0.1:5555", ctlweb))
+
+}

fedilogue/db.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"context"
+	"os"
+
+	"github.com/jackc/pgx/v4/pgxpool"
+)
+
+var pool *pgxpool.Pool
+
+func getDbPool() *pgxpool.Pool {
+	// Setup Database
+	dburl := "postgres://" + os.Getenv("POSTGRES_USER") + ":" + os.Getenv("POSTGRES_PASSWORD") + "@" + os.Getenv("POSTGRES_HOSTNAME") + "/" + os.Getenv("POSTGRES_DB")
+	pool, err := pgxpool.Connect(context.Background(), dburl)
+	if err != nil {
+		logFatal("Unable to connect to database:", err)
+	}
+	return pool
+}

fedilogue/fedilogue.go

@@ -0,0 +1,118 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	_ "net/http/pprof"
+	"regexp"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/microcosm-cc/bluemonday"
+	"git.farhan.codes/farhan/fedilogue/shared"
+)
+
+// Current instances
+var runninginstances map[string]shared.RunningInstance
+var ri_mutex = &sync.Mutex{}
+
+func startpprof() {
+	logInfo("Starting http/pprof on :7777")
+	logFatal(http.ListenAndServe("127.0.0.1:7777", nil))
+}
+
+func statusReportHandler() {
+	for {
+		StatusReport()
+		time.Sleep(time.Second * 60)
+	}
+}
+
+/* Tests:
+   - TestStatusReport_empty_run
+   - TestStatusReport_full_content
+*/
+func StatusReport() {
+	running := 0
+	keepalive := 0
+	unsupported := 0
+
+	mastodon := 0
+	pleroma := 0
+	misskey := 0
+	other := 0
+	ri_mutex.Lock()
+	for i, o := range runninginstances {
+		logDebug("Software ", o.Software, " Status: ", o.Status, " instance ", i)
+		if o.Status == 200 {
+			running = running + 1
+		} else if o.Status == 607 { // Keepalive
+			keepalive = keepalive + 1
+		} else if o.Status == 605 { // Unsupported instance
+			unsupported = unsupported + 1
+		}
+
+		if o.Software == "mastodon" && o.Status == 200 {
+			mastodon = mastodon + 1
+		} else if o.Software == "pleroma" && o.Status == 200 {
+			pleroma = pleroma + 1
+		} else if o.Software == "misskey" && o.Status == 200 {
+			misskey = misskey + 1
+		} else if o.Status == 200 {
+			other = other + 1
+		}
+	}
+	ri_mutex.Unlock()
+	logInfo("Running:", running, " Keepalive:", keepalive, " Unsupported:", unsupported, " Ma:", mastodon, ",P:", pleroma, ",Mi:", misskey, ",O:", other)
+}
+
+func main() {
+	// Initial Setup
+	logInit()
+	runninginstances = make(map[string]shared.RunningInstance)
+
+	getSettings()
+	go startpprof()
+
+	pool = getDbPool()
+
+	p = bluemonday.NewPolicy()
+	spaceReg = regexp.MustCompile(`[\s\t\.]+`)
+	removeHTMLReg = regexp.MustCompile(`<\/?\s*br\s*>`)
+	re = regexp.MustCompile("^https?://([^/]*)/(.*)$")
+	matchurl = regexp.MustCompile("http?s://[\\w\\-]+\\.[\\w\\-]+\\S*")
+	staggeredStartChan = make(chan bool)
+
+	// Start instances located in database
+	rows, err := pool.Query(context.Background(), "SELECT endpoint FROM instances")
+	if err != nil {
+		logErr("Unable to select from instances")
+		return
+	}
+	defer rows.Close()
+
+	go staggeredStart()
+	go statusReportHandler()
+
+	for rows.Next() {
+		var endpoint string
+		err = rows.Scan(&endpoint)
+		if err != nil {
+			logErr("Unable to iterate database, exiting.")
+			return
+		}
+		o, exists := GetRunner(endpoint)
+		if o.Banned == true {
+			continue // Banned instance
+		}
+		if exists == false {
+			go StartInstance(endpoint)
+		}
+	}
+
+	go startctl()
+	go webmain()
+
+	runtime.Goexit()
+}

fedilogue/fedilogue_test.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"git.farhan.codes/farhan/fedilogue/shared"
+)
+
+func TestStatusReport_empty_run(t *testing.T) {
+	// Empty Instances run
+	StatusReport()
+}
+
+func TestStatusReport_full_content(t *testing.T) {
+	defer func() {
+		runninginstances = map[string]shared.RunningInstance{}
+	}()
+	runninginstances = make(map[string]shared.RunningInstance)
+
+	identifier := 0
+	var endpoint string
+
+	test_instance_types := []string{"pleroma", "mastodon", "unknown", ""}
+	test_statuses := []int{shared.NEW_INSTANCE, shared.RUNNING, shared.UNAUTHORIZED, shared.FORBIDDEN, shared.NOT_FOUND, shared.UNPROCESSABLE_ENTITY, shared.TOOMANYREQUESTS, shared.INTERNAL_ERROR, shared.CLIENT_ISSUE, shared.ONION_PROTOCOL, shared.BAD_RESPONSE, shared.BAD_NODEINFO, shared.UNSUPPORTED_INSTANCE, shared.STREAM_ENDED, shared.KEEPALIVE}
+
+	for _, test_instance_type := range test_instance_types {
+		for _, test_status := range test_statuses {
+			a := shared.RunningInstance{}
+			endpoint = "endpoint" + strconv.Itoa(identifier) + ".test.com"
+			a.Client = BuildClient(endpoint)
+			a.Status = test_status
+			a.Recentactivities = shared.NewUniqueFifo(10)
+			a.Recentactors = shared.NewUniqueFifo(10)
+			a.Software = test_instance_type
+			a.Version = "0." + strconv.Itoa(identifier)
+			a.LastRun = time.Now().Format(time.RFC3339)
+			runninginstances[endpoint] = a
+			identifier = identifier + 1
+		}
+	}
+
+	StatusReport()
+
+}

fedilogue/follow.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/go-fed/httpsig"
+)
+
+func followInbox(inboxurl string) {
+	matchset := re.FindStringSubmatch(inboxurl)
+	if matchset == nil {
+		logWarn("Unable to unregex request", inboxurl)
+		return
+	}
+	inboxhostname := matchset[1]
+
+	keyBytes, err := ioutil.ReadFile("keys/private.pem")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	pemBlock, _ := pem.Decode(keyBytes)
+	if pemBlock == nil {
+		log.Fatal("Invalid PEM format")
+	}
+
+	privateKey, err := x509.ParsePKCS1PrivateKey(pemBlock.Bytes)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	jsonRequest := fmt.Sprintf(`{"@context":["https://www.w3.org/ns/activitystreams","https://%[2]s/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://%[2]s/relay","cc":[],"id":"https://%[2]s/activities/bd0614f5-371d-4bd2-88b3-1ee12f7bf42a","object":"%[1]s","state":"pending","to":["%[1]s"],"type":"Follow"}`, inboxurl, settings.Hostname)
+
+	//var jsonBytes []byte
+	jsonBytes := []byte(jsonRequest)
+	payload := bytes.NewReader(jsonBytes)
+
+	prefs := []httpsig.Algorithm{httpsig.RSA_SHA256}
+	digestAlgorithm := httpsig.DigestSha256
+	headers := []string{httpsig.RequestTarget, "date", "host"}
+	signer, _, err := httpsig.NewSigner(prefs, digestAlgorithm, headers, httpsig.Signature, 0)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	req, err := http.NewRequest("POST", inboxurl, payload)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+	if payload != nil {
+		req.Header.Add("content-type", "application/json")
+	}
+	req.Header.Add("date", time.Now().UTC().Format(http.TimeFormat))
+	req.Header.Add("host", inboxhostname)
+
+	keyID := "https://" + settings.Hostname + "/relay"
+
+	err = signer.SignRequest(privateKey, keyID, req, jsonBytes)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+	req.Header["Signature"][0] = strings.Replace(req.Header["Signature"][0], "algorithm=\"hs2019\"", "algorithm=\"rsa-sha256\"", 1)
+
+	customTransport := http.DefaultTransport.(*http.Transport).Clone()
+	customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	client := &http.Client{Timeout: 10 * time.Second, Transport: customTransport}
+
+	//	client := &http.Client{Timeout: 10 * time.Second}
+	res, err := client.Do(req)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer res.Body.Close()
+}

fedilogue/go.mod

@@ -0,0 +1,29 @@
+module git.farhan.codes/farhan/fedilogue/fedilogue
+
+go 1.23.5
+
+require (
+	git.farhan.codes/farhan/fedilogue/shared v0.0.0-20250124035748-fa4db6191391
+	github.com/go-fed/httpsig v1.1.0
+	github.com/google/uuid v1.6.0
+	github.com/gorilla/websocket v1.5.3
+	github.com/jackc/pgx/v4 v4.18.3
+	github.com/microcosm-cc/bluemonday v1.0.27
+)
+
+require (
+	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/gorilla/css v1.0.1 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.14.3 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.3 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgtype v1.14.0 // indirect
+	github.com/jackc/puddle v1.3.0 // indirect
+	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+)

fedilogue/go.sum

@@ -0,0 +1,187 @@
+git.farhan.codes/farhan/fedilogue/shared v0.0.0-20250124035748-fa4db6191391 h1:/pt4xgnZGxS/pC+CpRQ498gfbOy1E3o16eGKch4AR10=
+git.farhan.codes/farhan/fedilogue/shared v0.0.0-20250124035748-fa4db6191391/go.mod h1:KT2OwXD90rPF8qPL/pJVg4WRwq4qvLQcMnoyldckXXw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
+github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
+github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
+github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
+github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
+github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
+github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
+github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
+github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
+github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
+github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
+github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
+github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w=
+github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM=
+github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
+github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
+github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
+github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c=
+github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
+github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
+github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag=
+github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
+github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
+github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
+github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
+github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
+github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw=
+github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
+github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
+github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
+github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
+github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
+github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA=
+github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw=
+github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
+github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
+github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
+github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=

fedilogue/instance.go

@@ -0,0 +1,282 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net"
+	"net/http"
+	"time"
+
+	"git.farhan.codes/farhan/fedilogue/shared"
+)
+
+var staggeredStartChan chan bool
+
+func DoTries(o *shared.RunningInstance, req *http.Request) (*http.Response, error) {
+	var resp *http.Response
+	var err error
+
+	for tries := 0; tries < 10; tries++ {
+		resp, err = o.Client.Do(req)
+		if err != nil {
+			// URL.Scheme, Host, Path Opaque
+			logWarn("Failure connecting to "+req.URL.Scheme+"://"+req.URL.Host+req.URL.Path+", attempt ", tries+1, ", sleeping for 5 minutes: ", err)
+			time.Sleep(time.Minute * 5)
+			continue
+		}
+		break
+	}
+	return resp, err
+}
+
+func BuildClient(endpoint string) http.Client {
+	logDebug("BuildClient for ", endpoint)
+	// Test: TestBuildClient, TestBuildClientProxy
+	/* The seemingly unused 'endpoint' variable is for proxying based on endpoint, ie for Tor */
+	tr := &http.Transport{
+		MaxIdleConns:    2,
+		IdleConnTimeout: 3600 * time.Second,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+	}
+	client := http.Client{Transport: tr}
+
+	return client
+}
+
+func GetRunner(endpoint string) (shared.RunningInstance, bool) {
+	// Tests: TestGetRunnerNonExist, TestGetRunnerExists
+	ri_mutex.Lock()
+	o, exists := runninginstances[endpoint]
+
+	if exists == false {
+		o = shared.RunningInstance{}
+		selectRet := pool.QueryRow(context.Background(), "SELECT banned, alwaysbot FROM instances WHERE endpoint = $1", endpoint)
+		err := selectRet.Scan(&o.Banned, &o.Alwaysbot)
+		if err != nil {
+			logDebug("Did not find instance in database: ", endpoint)
+		}
+		if o.Banned == true {
+			logInfo("Banned instance: ", endpoint)
+		} else {
+			logDebug("Building runner for: ", endpoint)
+			o.Client = BuildClient(endpoint)
+			o.Status = shared.KEEPALIVE
+			o.Recentactivities = shared.NewUniqueFifo(10)
+			o.Recentactors = shared.NewUniqueFifo(10)
+		}
+		runninginstances[endpoint] = o
+	}
+	ri_mutex.Unlock()
+
+	return o, exists
+}
+
+func UpdateRunner(endpoint string, o shared.RunningInstance) {
+	// Tests: None necessary
+	ri_mutex.Lock()
+	runninginstances[endpoint] = o
+	ri_mutex.Unlock()
+}
+
+func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningInstance {
+	/* Checking order
+	 * Mastodon/Pleroma/Misskey
+	 * Um..nothing else yet
+	 */
+	logDebug("GetInstanceInfo for ", endpoint)
+	var nodeinfo shared.NodeInfo
+	pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"
+
+	// Checking Mastodon and Pleroma (with .json)
+	logDebug("GetInstanceInfo: For " + endpoint + " Mastodon/Pleroma URL: " + pleromastodon_nodeinfo_uri)
+	reqjson, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil)
+	reqjson.Header.Set("User-Agent", "Tusky")
+
+	pleromastodon_api_resp, err := DoTries(&o, reqjson)
+	if err != nil {
+		o.Software = "Unsupported"
+		return o
+	} else {
+		defer pleromastodon_api_resp.Body.Close()
+	}
+
+	if pleromastodon_api_resp.StatusCode == 200 {
+		err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo)
+		if err == nil {
+			o.Software = nodeinfo.Software.Name
+			o.Version = nodeinfo.Software.Version
+			o.LastRun = time.Now().Format(time.RFC3339)
+			defer pleromastodon_api_resp.Body.Close()
+			return o
+		}
+	}
+
+	// Checking for Misskey (without .json)
+	misskey_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0"
+	logDebug("GetInstanceInfo: For " + endpoint + " MissKey URL: " + misskey_nodeinfo_uri)
+	req, _ := http.NewRequest("GET", misskey_nodeinfo_uri, nil)
+	req.Header.Set("User-Agent", "Tusky")
+
+	misskey_api_resp, err := DoTries(&o, req)
+	if err != nil {
+		o.Software = "Unsupported"
+		return o
+	} else {
+		defer misskey_api_resp.Body.Close()
+	}
+
+	if misskey_api_resp.StatusCode == 200 {
+		err = json.NewDecoder(misskey_api_resp.Body).Decode(&nodeinfo)
+		if err == nil {
+			o.Software = nodeinfo.Software.Name
+			o.Version = nodeinfo.Software.Version
+			o.LastRun = time.Now().Format(time.RFC3339)
+			defer misskey_api_resp.Body.Close()
+			return o
+		}
+	}
+
+	// Check for Lemmy (With Json)
+	lemmy_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.1"
+	logDebug("GetInstanceInfo: For " + endpoint + " Lemmy URL: " + lemmy_nodeinfo_uri)
+	req, _ = http.NewRequest("GET", lemmy_nodeinfo_uri, nil)
+	req.Header.Set("User-Agent", "Tusky")
+
+	lemmy_api_resp, err := DoTries(&o, req)
+	if err != nil {
+		o.Software = "Unsupported"
+		return o
+	} else {
+		defer lemmy_api_resp.Body.Close()
+	}
+
+	if lemmy_api_resp.StatusCode == 200 {
+		err = json.NewDecoder(lemmy_api_resp.Body).Decode(&nodeinfo)
+		if err == nil {
+			logDebug("Found a new Lemmy instance: " + endpoint)
+			o.Software = nodeinfo.Software.Name
+			o.Version = nodeinfo.Software.Version
+			o.LastRun = time.Now().Format(time.RFC3339)
+			defer lemmy_api_resp.Body.Close()
+			return o
+		}
+	}
+
+	// Unsupported Software
+	o.Software = "Unsupported"
+	o.Version = "Unknown"
+
+	return o
+}
+
+func LogInstance(endpoint string, o shared.RunningInstance) bool {
+	logDebug("LogInstance: ", endpoint)
+	selectRet := pool.QueryRow(context.Background(), "SELECT FROM instances WHERE endpoint = $1", endpoint)
+	err := selectRet.Scan()
+	if err == nil {
+		return true // Endpoint already in database, continuing
+	}
+
+	_, err = pool.Exec(context.Background(), "INSERT INTO instances (endpoint, state, software) VALUES($1, $2, $3)", endpoint, "", o.Software)
+	if err != nil {
+		logWarn("Error inserting ", endpoint+" into `instances`: ", err)
+		return true
+	}
+
+	return false
+}
+
+func CheckInstance(newinstance string, callerEndpoint string) {
+	logDebug("checkInstance: ", newinstance)
+	if settings.Crawl == true {
+		// Skip over this if its the same as the endpoint or empty
+		if newinstance == callerEndpoint || newinstance == "" {
+			return
+		}
+
+		var err error
+		for attempt := 0; attempt > 5; attempt = attempt + 1 {
+			_, err = net.LookupHost(newinstance)
+			if err != nil {
+				logDebug("Unable to resolve "+newinstance+" attempt ", attempt, "/5. Sleeping for 30 seconds")
+				time.Sleep(time.Second * 30)
+				continue
+			}
+			break
+		}
+		if err != nil {
+			logWarn("Unable to resolve ", newinstance, " after 5 attempts, giving up: ", err)
+			return
+		}
+
+		// Skip over this if its the same as the endpoint
+		if newinstance == callerEndpoint {
+			return
+		}
+
+		// Going forward, this might be merged into GetRunner
+		ri_mutex.Lock()
+		o, exists := runninginstances[newinstance]
+		if exists == false || o.Status == shared.KEEPALIVE {
+			m := shared.RunningInstance{}
+			m.Client = BuildClient(newinstance)
+			m.Recentactivities = shared.NewUniqueFifo(10)
+			m.Recentactors = shared.NewUniqueFifo(10)
+			runninginstances[newinstance] = m
+			go StartInstance(newinstance)
+		}
+		ri_mutex.Unlock()
+	} else {
+		logDebug("Not crawling into ", newinstance)
+	}
+
+}
+
+func staggeredStart() {
+	for {
+	_:
+		<-staggeredStartChan
+		time.Sleep(500 * time.Millisecond)
+	}
+}
+
+func StartInstance(endpoint string) {
+	staggeredStartChan <- true
+	logInfo("Starting " + endpoint)
+
+	// Check if exists. If so, get the object. If not, create it
+	o, _ := GetRunner(endpoint)
+	if o.Banned == true {
+		logInfo("Ignoring banned instance: ", endpoint)
+		return // banned instance
+	}
+
+	o = GetInstanceInfo(endpoint, o)
+	UpdateRunner(endpoint, o)
+	LogInstance(endpoint, o)
+
+	if o.Software == "pleroma" {
+		logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
+		o.CaptureType = "Stream"
+		UpdateRunner(endpoint, o)
+		StreamPleroma(endpoint)
+	} else if o.Software == "mastodon" {
+		logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
+		o.CaptureType = "Stream"
+		UpdateRunner(endpoint, o)
+		StreamMastodon(endpoint, &o)
+	} else if o.Software == "misskey" {
+		logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
+		o.CaptureType = "Stream"
+		UpdateRunner(endpoint, o)
+		StreamMisskey(endpoint)
+	} else {
+		o.Status = 605
+		UpdateRunner(endpoint, o)
+		logConn("Unsupported endpoint " + endpoint)
+	}
+}

fedilogue/instance_test.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"net"
+	"net/http"
+	"reflect"
+	"testing"
+	"time"
+
+	"git.farhan.codes/farhan/fedilogue/shared"
+)
+
+func TestBuildClient(t *testing.T) {
+	tr := &http.Transport{
+		MaxIdleConns:    2,
+		IdleConnTimeout: 3600 * time.Second,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+	}
+	want := http.Client{Transport: tr}
+	have := BuildClient("testdomain.com")
+
+	if reflect.DeepEqual(want, have) {
+		t.Fatalf("TestBuildClient client different from expected.")
+	}
+}
+
+func TestBuildClientProxy(t *testing.T) {
+	// Currently not implemented
+}
+
+func TestGetRunnerNonExist(t *testing.T) {
+	// Currently not implemented
+}
+
+func TestGetRunnerExists(t *testing.T) {
+	defer func() {
+		runninginstances = map[string]shared.RunningInstance{}
+	}()
+
+	want_o := shared.RunningInstance{}
+	want_o.Client = BuildClient("some-non-existent-domain.tld")
+	want_o.Status = shared.KEEPALIVE
+	want_o.Recentactivities = shared.NewUniqueFifo(10)
+	want_o.Recentactors = shared.NewUniqueFifo(10)
+	runninginstances["some-non-existent-domain.tld"] = want_o
+
+	want_exists := true
+	_, have_exists := GetRunner("some-non-existent-domain.tld")
+
+	if have_exists != want_exists {
+		t.Fatalf("TestGetRunnerBlank expected %v, got %v", want_exists, have_exists)
+	}
+	//	if reflect.DeepEqual(want_o, have_o) {
+	//		t.Fatalf("TestGetRunnerExists failed, should have the same value")
+	//	}
+}

fedilogue/log.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"log"
+	"os"
+)
+
+var (
+	_logInfo  *log.Logger
+	_logErr   *log.Logger
+	_logWarn  *log.Logger
+	_logConn  *log.Logger
+	_logDebug *log.Logger
+	_logFatal *log.Logger
+)
+
+func logInit() {
+	_logInfo = log.New(os.Stdout, "INFO: ", log.Lmsgprefix|log.Ldate|log.Ltime)
+	_logErr = log.New(os.Stdout, "ERR: ", log.Lmsgprefix|log.Ldate|log.Ltime)
+	_logWarn = log.New(os.Stdout, "WARN: ", log.Lmsgprefix|log.Ldate|log.Ltime)
+	_logConn = log.New(os.Stdout, "CONN: ", log.Lmsgprefix|log.Ldate|log.Ltime)
+	_logDebug = log.New(os.Stdout, "DEBUG: ", log.Lmsgprefix|log.Ldate|log.Ltime)
+	_logFatal = log.New(os.Stdout, "FATAL: ", log.Lmsgprefix|log.Ldate|log.Ltime)
+}
+
+func logInfo(s ...interface{}) {
+	if settings.LogLevel >= 1 {
+		_logInfo.Print(s...)
+	}
+}
+
+func logErr(s ...interface{}) {
+	if settings.LogLevel >= 2 {
+		_logErr.Print(s...)
+	}
+}
+
+func logWarn(s ...interface{}) {
+	if settings.LogLevel >= 3 {
+		_logWarn.Print(s...)
+	}
+}
+
+func logConn(s ...interface{}) {
+	if settings.LogLevel >= 4 {
+		_logConn.Print(s...)
+	}
+}
+
+func logDebug(s ...interface{}) {
+	if settings.LogLevel >= 5 {
+		_logDebug.Print(s...)
+	}
+}
+
+func logFatal(s ...interface{}) {
+	_logFatal.Fatal(s...)
+}

fedilogue/oauth.go

@@ -0,0 +1,210 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	"git.farhan.codes/farhan/fedilogue/shared"
+)
+
+type OAuth struct {
+	Access_token  string `"json:access_token"`
+	Created_at    int    `"json:created_at"`
+	Expires_in    int64  `"json:Expires_in"`
+	Refresh_token string `"json:refresh_token"`
+}
+
+type authError struct {
+	msg string
+}
+
+func (e *authError) Error() string {
+	return e.msg
+}
+
+func register_client(endpoint string, o *shared.RunningInstance) error {
+	requestBodymap, _ := json.Marshal(map[string]string{
+		"client_name":   "Tusky", // Hard-coded in for now...
+		"scopes":        "read write follow push",
+		"redirect_uris": "urn:ietf:wg:oauth:2.0:oob",
+	})
+	requestBodybytes := bytes.NewBuffer(requestBodymap)
+
+	api_base_apps := "https://" + endpoint + "/api/v1/apps"
+
+	resp, err := o.Client.Post(api_base_apps, "application/json", requestBodybytes)
+	if err != nil {
+		logErr("Unable to connect to "+api_base_apps+" ", err)
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		logErr("Unable to read HTTP response: ", err)
+		o.Client_id = ""
+		o.Client_secret = ""
+		return err
+	}
+
+	bodymap := make(map[string]string)
+	err = json.Unmarshal(body, &bodymap)
+	if err != nil {
+		logErr("Unable to parse response from "+endpoint+": ", err)
+		o.Client_id = ""
+		o.Client_secret = ""
+		return err
+	}
+
+	client_file := "clients/" + endpoint
+
+	f, err := os.Create("clients/" + endpoint)
+	if err != nil {
+		logErr("Unable to create "+client_file+": ", err)
+		o.Client_id = ""
+		o.Client_secret = ""
+		return err
+	}
+	defer f.Close()
+
+	_, err = io.WriteString(f, bodymap["client_id"]+"\n")
+	if err != nil {
+		logErr("Unable to write client_id line to file "+client_file+": ", err)
+		o.Client_id = bodymap["client_id"]
+		o.Client_secret = bodymap["client_secret"]
+		return nil
+	}
+	_, err = io.WriteString(f, bodymap["client_secret"]+"\n")
+	if err != nil {
+		logErr("Unable to write client_secret to file "+client_file+": ", err)
+		o.Client_id = bodymap["client_id"]
+		o.Client_secret = bodymap["client_secret"]
+		return nil
+	}
+
+	o.Client_id = bodymap["client_id"]
+	o.Client_secret = bodymap["client_secret"]
+	return nil
+}
+
+func get_client(endpoint string, o *shared.RunningInstance) error {
+	var err error
+	client_file := "clients/" + endpoint
+	_, err = os.Stat(client_file)
+	if os.IsNotExist(err) == false { // The file exists
+		f, err := os.Open(client_file)
+		if err != nil {
+			logErr("Unable to open " + client_file + ", creating new client")
+			return err
+		}
+		defer f.Close()
+
+		rd := bufio.NewReader(f)
+
+		client_id_bin, _, err := rd.ReadLine()
+		o.Client_id = string(client_id_bin)
+		if err != nil {
+			logErr("Unable to read client_id line of " + client_file + ", building new client")
+			return err
+		}
+		client_secret_bin, _, err := rd.ReadLine()
+		o.Client_secret = string(client_secret_bin)
+		if err != nil {
+			logErr("Unable to read client_secret line of " + client_file + ", building new client")
+			return err
+		}
+
+		return nil
+	} else {
+		return register_client(endpoint, o)
+	}
+
+	return nil
+}
+
+func oauth_login(endpoint string, o *shared.RunningInstance, username string, password string) (OAuth, error) {
+	authMap, err := json.Marshal(map[string]string{
+		"username":      username,
+		"password":      password,
+		"redirect_uri":  "urn:ietf:wg:oauth:2.0:oob",
+		"grant_type":    "password",
+		"client_name":   "Tusky",
+		"scope":         "read write follow push",
+		"client_id":     o.Client_id,
+		"client_secret": o.Client_secret,
+	})
+
+	if err != nil {
+		logErr("Unable to create Authentication map for " + endpoint)
+		return OAuth{}, err
+	}
+
+	authMapbytes := bytes.NewBuffer(authMap)
+
+	resp, err := http.Post("https://"+endpoint+"/oauth/token", "application/json", authMapbytes)
+	if err != nil {
+		logErr("Cannot connect to "+endpoint+": ", err)
+		return OAuth{}, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		logErr("Unable to read response data for "+endpoint+": ", err)
+		return OAuth{}, err
+	}
+
+	if resp.StatusCode == 400 {
+		logErr("Unable to authenticate to " + endpoint)
+		return OAuth{}, &authError{"Authentication error"}
+	}
+
+	oauthData := OAuth{}
+	err = json.Unmarshal(body, &oauthData)
+	if err != nil {
+		logErr("Unable to parse json data for "+endpoint+": ", err)
+		return OAuth{}, err
+	}
+
+	return oauthData, nil
+}
+
+func oauth_refresh(endpoint string, client_id string, client_secret string, refresh_token string) (OAuth, error) {
+	authMap, _ := json.Marshal(map[string]string{
+		"redirect_uri":  "urn:ietf:wg:oauth:2.0:oob",
+		"grant_type":    "refresh_token",
+		"scope":         "read write follow push",
+		"refresh_token": refresh_token,
+		"client_id":     client_id,
+		"client_secret": client_secret,
+	})
+
+	authMapbytes := bytes.NewBuffer(authMap)
+
+	resp, err := http.Post("https://"+endpoint+"/oauth/token", "application/json", authMapbytes)
+	if err != nil {
+		logErr("Unable to connect to "+endpoint+": ", err)
+		return OAuth{}, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		logErr("Unable to read response data for "+endpoint+": ", err)
+		return OAuth{}, err
+	}
+
+	oauthData := OAuth{}
+	err = json.Unmarshal(body, &oauthData)
+	if err != nil {
+		logErr("Unable to parse json data for "+endpoint+": ", err)
+		return oauthData, err
+	}
+
+	return oauthData, nil
+}

fedilogue/poll.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"git.farhan.codes/farhan/fedilogue/shared"
+)
+
+type ImageData struct {
+	Type string `"json:type"`
+	Url  string `"json:url"`
+}
+
+type PublicKeyData struct {
+	Id           string `"json:id"`
+	Owner        string `"json:owner"`
+	PublicKeyPem string `"json:publicKeyPem"`
+}
+
+type UserInfo struct {
+	Id                string        `"json:id"`
+	Type              string        `"json:type"`
+	Following         string        `"json:following"`
+	Followers         string        `"json:followers"`
+	Inbox             string        `"json:inbox"`
+	Outbox            string        `"json:outbox"`
+	Featured          string        `"json:featured"`
+	PreferredUsername string        `"json:preferredUsername"`
+	PublicKey         PublicKeyData `"json:publicKeyPem"`
+	Name              string        `"json:name"`
+	Summary           string        `"json:summary"`
+	Url               string        `"json:Url"`
+	Icon              ImageData     `"json:icon"`
+	Image             ImageData     `"json:image"`
+}
+
+type PostInfo struct {
+	Id        string `"json:id"`
+	Type      string `"json:type"`
+	Published string `"json:published"`
+	Url       string `"json:Url"`
+	Content   string `"json:content"`
+}
+
+func PollMastodonPleroma(endpoint string, o *shared.RunningInstance) {
+	newactivities := make([]shared.ReportActivity, 0)
+
+	min_id := ""
+
+	parsing_error := 0
+	use_auth := false
+
+	var last_refresh int64
+	var client_id string
+	var client_secret string
+	var oauthData OAuth
+
+	for {
+		ri_mutex.Lock()
+		m := runninginstances[endpoint]
+		ri_mutex.Unlock()
+
+		api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&since_id=" + min_id
+		req, err := http.NewRequest("GET", api_timeline, nil)
+		req.Header.Set("User-Agent", "Tusky")
+		if err != nil {
+			logFatal("Unable to create new request for "+endpoint+": ", err)
+			return
+		}
+
+		if use_auth == true {
+			if time.Now().Unix() > last_refresh+oauthData.Expires_in {
+				oauthData, err = oauth_refresh(endpoint, client_id, client_secret, oauthData.Refresh_token)
+				if err != nil {
+					logWarn("Unable to refresh oauth token for "+endpoint+": ", err)
+					return
+				}
+				last_refresh = time.Now().Unix()
+			}
+			req.Header.Add("Authorization", oauthData.Access_token)
+		}
+
+		m.LastRun = time.Now().Format(time.RFC3339)
+		resp, err := DoTries(o, req)
+		if err != nil {
+			m.Status = shared.CLIENT_ISSUE
+			ri_mutex.Lock()
+			runninginstances[endpoint] = m
+			ri_mutex.Unlock()
+			logWarn("Giving up on "+endpoint+": ", err.Error())
+			return
+		}
+
+		if resp.StatusCode == shared.TOOMANYREQUESTS { // Short Delay, 30 seconds
+			logWarn("Delaying "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay")
+			_, _ = ioutil.ReadAll(resp.Body)
+			resp.Body.Close() // Release as soon as done
+			m.Status = resp.StatusCode
+			ri_mutex.Lock()
+			runninginstances[endpoint] = m
+			ri_mutex.Unlock()
+			time.Sleep(time.Second * 30)
+			continue
+		} else if resp.StatusCode == shared.INTERNAL_ERROR { // Longer delay, 1 hour
+			logWarn("Suspending "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay")
+			_, _ = ioutil.ReadAll(resp.Body)
+			resp.Body.Close() // Release as soon as done
+			m.Status = 765
+			ri_mutex.Lock()
+			runninginstances[endpoint] = m
+			ri_mutex.Unlock()
+			time.Sleep(time.Second * 3600)
+			continue
+		} else if resp.StatusCode != 200 { // Crash
+			logErr("Terminating "+endpoint+", gave status ", resp.StatusCode)
+			_, _ = ioutil.ReadAll(resp.Body)
+			resp.Body.Close() // Release as soon as done
+			m.Status = resp.StatusCode
+			ri_mutex.Lock()
+			runninginstances[endpoint] = m
+			ri_mutex.Unlock()
+			return
+		}
+
+		err = json.NewDecoder(resp.Body).Decode(&newactivities)
+		resp.Body.Close() // Release as soon as done
+		if err != nil {
+			if parsing_error > 5 {
+				m.Status = shared.BAD_RESPONSE
+				ri_mutex.Lock()
+				runninginstances[endpoint] = m
+				ri_mutex.Unlock()
+				logErr("Giving up on " + endpoint + " after 5 unmarshal errors.")
+				return
+			}
+			parsing_error = parsing_error + 1
+			time.Sleep(time.Second * 30)
+		}
+
+		m.Status = shared.RUNNING
+		ri_mutex.Lock()
+		runninginstances[endpoint] = m
+		ri_mutex.Unlock()
+
+		for _, newactivity := range newactivities {
+			go check_activity(newactivity.Uri)
+			matchset := re.FindStringSubmatch(newactivity.Uri)
+			if matchset != nil {
+				newinstance := matchset[1]
+
+				// Check min_id
+				if newactivity.Id > min_id {
+					min_id = newactivity.Id
+				}
+
+				go CheckInstance(newinstance, endpoint)
+			}
+		}
+		time.Sleep(time.Second * 10)
+	}
+}

fedilogue/retrieve.go

@@ -0,0 +1,287 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"html"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/microcosm-cc/bluemonday"
+)
+
+var p *bluemonday.Policy
+var spaceReg *regexp.Regexp
+var removeHTMLReg *regexp.Regexp
+var re *regexp.Regexp
+var matchurl *regexp.Regexp
+
+type ImageType struct {
+	Url string `json:"url"`
+}
+
+type PublicKeyType struct {
+	PublicKeyPem string `json:"publicKeyPem"`
+}
+
+type ActorJson struct {
+	id                int
+	Uri               string        `json:"id"`
+	Type              string        `json:"type"`
+	Inbox             string        `json:"inbox"`
+	Outbox            string        `json:"outbox"`
+	Followers         string        `json:"followers"`
+	Following         string        `json:"following"`
+	Url               string        `json:"url"`
+	PreferredUsername string        `json:"preferredUsername"`
+	Name              string        `json:"name"`
+	Summary           string        `json:"summary"`
+	Icon              ImageType     `json:"icon"`
+	Image             ImageType     `json:"image"`
+	PublicKey         PublicKeyType `json:"publicKey"`
+
+	bot      bool
+	instance string
+}
+
+type TagType struct {
+	Type string `json:"type"`
+	Name string `json:"name"`
+}
+
+type PostJson struct {
+	id        int
+	Uri       string `json:"id"`
+	InReplyTo string `json:"inReplyTo"`
+
+	normalized string
+	receivedAt time.Time `json:"created_at"`
+
+	Content      string    `json:"content"`
+	Conversation string    `json:"conversation"`
+	Published    time.Time `json:"published"`
+	Summary      string    `json:"summary"`
+	Tag          []TagType `json:"tag"`
+	To           []string  `json:"to"`
+	Type         string    `json:"type"`
+
+	Actor        string `json:"actor"`
+	AttributedTo string `json:"attributedTo"`
+
+	bot      bool
+	instance string
+}
+
+func check_activity(uri string) {
+	logDebug("Retrieving: " + uri)
+	var activityjson PostJson
+
+	// Ignore invalid URIs
+	endslash := strings.Index(uri[8:], "/")
+	if endslash == -1 {
+		return
+	}
+	activityjson.instance = uri[8 : endslash+8]
+
+	o, _ := GetRunner(activityjson.instance)
+	if o.Banned == true {
+		logDebug("Ignoring banned instance: ", uri)
+		return // Banned instance
+	}
+
+	// Check if there were any recent requests on this
+	o.Recentactivities.Mu.Lock()
+	i, _ := o.Recentactivities.Contains(uri)
+	if i != -1 {
+		logDebug("Ignoring cached recent request: ", uri)
+		o.Recentactivities.Mu.Unlock()
+		return
+	}
+
+	o.Recentactivities.Add(uri, "") // Added blank entry
+	o.Recentactivities.Mu.Unlock()
+	var jsondocument string
+
+	selectRet := pool.QueryRow(context.Background(), "SELECT FROM activities WHERE document->>'id' = $1", uri)
+	err := selectRet.Scan()
+	if err == nil {
+		logDebug("Already in database, ignoring: ", uri)
+		return
+	}
+
+	req, _ := http.NewRequest("GET", uri, nil)
+	req.Header.Set("User-Agent", "Tusky")
+	req.Header.Add("Accept", "application/ld+json")
+
+	resp, err := DoTries(&o, req)
+	if err != nil {
+		logDebug("Gave up after multiple tries: ", uri)
+		return
+	}
+
+	if resp.StatusCode != 200 {
+		logDebug("Non-200 response code for ", uri, " was ", resp.StatusCode)
+		resp.Body.Close()
+		return
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		logDebug("Failed to read the reply: ", uri)
+		return
+	}
+	resp.Body.Close()
+	jsondocument = string(body)
+	err = json.Unmarshal(body, &activityjson)
+	if err != nil {
+		logDebug("Failed to Unmarshal, err: ", err, " uri: ", uri)
+		return
+	}
+
+	if activityjson.InReplyTo != "" && activityjson.InReplyTo != uri {
+		if activityjson.InReplyTo != uri {
+			go check_activity(activityjson.InReplyTo)
+		}
+	}
+
+	// If AttributedTo is blank, this is likely an authentication failure
+	// For now, skip it...
+	if activityjson.AttributedTo == "" {
+		logDebug("AttributedTo field is blank, dropping for ", uri)
+		return
+	}
+
+	// This must be done BEFORE the `INSERT INTO activities'` below
+	actorjson := check_actor(activityjson.AttributedTo)
+	if actorjson == nil {
+		logDebug("Failed to add actor, dropping post: ", uri)
+		return
+	}
+	if actorjson.bot || o.Alwaysbot {
+		activityjson.bot = true
+	}
+
+	activityjson.normalized = removeHTMLReg.ReplaceAllString(activityjson.Content, " ")
+	activityjson.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(activityjson.normalized)))
+	activityjson.normalized = matchurl.ReplaceAllString(activityjson.normalized, "")
+	activityjson.normalized = spaceReg.ReplaceAllString(activityjson.normalized, " ")
+
+	var hashtags []string
+	for _, tag := range activityjson.Tag {
+		if tag.Type == "Hashtag" {
+			hashtags = append(hashtags, strings.ToLower(tag.Name))
+		}
+	}
+	_, err = pool.Exec(context.Background(), "INSERT INTO activities (document, normalized, instance, hashtags, bot) VALUES($1, $2, $3, $4, $5)", jsondocument, activityjson.normalized, activityjson.instance, hashtags, activityjson.bot)
+	if err != nil {
+		logWarn("Error inserting ", uri, " into `activities`: ", err)
+		return
+	}
+
+	for _, to := range activityjson.To {
+		if to != "https://www.w3.org/ns/activitystreams#Public" && to != "" {
+			if strings.HasSuffix(to, "/followers") {
+				// This check is very much a bad solution, may consider removing the entire for-loop
+				continue
+			}
+			go check_actor(to)
+		}
+	}
+
+}
+
+/* Test: TestCheck_actor */
+func check_actor(uri string) *ActorJson {
+	actorjson := &ActorJson{}
+
+	if len(uri) <= 7 {
+		return nil // Bad actor
+	}
+
+	endslash := strings.Index(uri[8:], "/")
+	if endslash == -1 {
+		return nil // Bad actor
+	}
+	actorjson.instance = uri[8 : endslash+8]
+
+	// Check if there were any recent requests on this
+	o, _ := GetRunner(actorjson.instance)
+	if o.Banned {
+		logDebug("Banned actor: ", uri)
+		return nil // Banned actor
+	}
+	o.Recentactors.Mu.Lock()
+	i, cachedactorjson := o.Recentactors.Contains(uri)
+	if i != -1 {
+		o.Recentactors.Mu.Unlock()
+		cachedactorjson := cachedactorjson.(*ActorJson)
+		return cachedactorjson
+	}
+	o.Recentactors.Mu.Unlock()
+
+	selectRet := pool.QueryRow(context.Background(), "SELECT document FROM actors WHERE document->>'id' = $1", uri)
+	err := selectRet.Scan(&actorjson)
+	if err == nil {
+		return actorjson // Actor already in database, good!
+	}
+
+	req, _ := http.NewRequest("GET", uri, nil)
+	req.Header.Set("User-Agent", "Tusky")
+	req.Header.Add("Accept", "application/ld+json")
+
+	var resp *http.Response
+	tries := 0
+	for {
+		resp, err = o.Client.Do(req)
+		if err != nil {
+			if tries > 10 {
+				logErr("Unable to connect to " + uri + " attempt 10/10, giving up.")
+				return nil // Unable to connect to host after 10 attempts
+			}
+			logWarn("Unable to connect to "+uri+", attempt ", tries+1, "+/10 sleeping for 30 seconds.")
+			time.Sleep(time.Second * 30)
+			tries = tries + 1
+			continue
+		}
+		break
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		logWarn("Unable to read body from ", uri)
+		return nil // Unable to read body of message
+	}
+	resp.Body.Close()
+
+	jsondocument := string(body)
+
+	err = json.Unmarshal(body, &actorjson)
+	if err != nil {
+		logWarn("Unable to unmarshal body from ", uri)
+		return nil // Unable to unmarshal body of message
+	}
+	o.Recentactors.Mu.Lock()
+	o.Recentactors.Add(uri, actorjson)
+	o.Recentactors.Mu.Unlock()
+
+	var bot bool
+	if actorjson.Type == "Service" {
+		actorjson.bot = true
+	} else {
+		actorjson.bot = o.Alwaysbot // default on host's classification
+	}
+
+	_, err = pool.Exec(context.Background(), "INSERT INTO actors (document, instance, bot) VALUES($1, $2, $3)", jsondocument, actorjson.instance, bot)
+	if err != nil {
+		logWarn("Error inserting ", uri, " into `actors`: ", err)
+		return nil // Unable to insert actor
+	}
+
+	o.Recentactors.Mu.Lock()
+	o.Recentactors.Add(uri, actorjson)
+	o.Recentactors.Mu.Unlock()
+	return actorjson // Successful
+}

fedilogue/retrieve_test.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestCheck_actor(t *testing.T) {
+	// Currently not implemented
+}

fedilogue/stream.go

@@ -0,0 +1,294 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/gorilla/websocket"
+	"git.farhan.codes/farhan/fedilogue/shared"
+)
+
+type MisskeyReply struct {
+	Type string           `json:"type"`
+	Body MisskeyReplyBody `json:"body"`
+}
+
+type MisskeyReplyBody struct {
+	Channel string          `json:"channel"`
+	Type    string          `json:"type"`
+	Body    MisskeyNoteBody `json:"body"`
+}
+
+type MisskeyNoteBody struct {
+	Uri string `json:"uri"` // Remote Note
+	Id  string `json:"id"`  // Local note
+}
+
+type MisskeyRequest struct {
+	Type string             `json:"type"`
+	Body MisskeyRequestBody `json:"body"`
+}
+
+type MisskeyRequestBody struct {
+	Channel string `json:"channel"`
+	Id      string `json:"id"`
+	Params  MisskeyRequestParams
+}
+
+type MisskeyRequestParams struct {
+}
+
+func StreamMisskey(endpoint string) {
+	logDebug("Stream Misskey for ", endpoint)
+	u := url.URL{
+		Scheme: "wss",
+		Host:   endpoint,
+		Path:   "/streaming",
+	}
+
+	var misskeyrequest MisskeyRequest
+	misskeyrequest.Type = "connect"
+	misskeyrequest.Body.Channel = "globalTimeline"
+	misskeyrequest.Body.Id = uuid.New().String()
+
+	for {
+		misskeyrequeststream, err := json.Marshal(misskeyrequest)
+		if err != nil {
+			panic(err)
+		}
+
+		for {
+			// Create a new WebSocket connection.
+			ws, _, err := websocket.DefaultDialer.Dial(u.String(), nil)
+			if err != nil {
+				logErr("Error dialing misskey webSocket to ", endpoint, " :", err)
+				return
+			}
+			logDebug("Misskey websocket connection created: ", endpoint)
+
+			ri_mutex.Lock()
+			m := runninginstances[endpoint]
+			m.Status = shared.RUNNING
+			m.LastRun = "Streaming"
+			runninginstances[endpoint] = m
+			ri_mutex.Unlock()
+
+			// Send a message to the server.
+			err = ws.WriteMessage(websocket.TextMessage, misskeyrequeststream)
+			if err != nil {
+				logErr("Error sending misskey channel subscription: ", endpoint)
+				return
+			}
+			logDebug("Successfully sent misskey channel subscription: ", endpoint)
+
+			// Read a message from the server.
+			for {
+				logDebug("Starting Misskey Stream loop for ", endpoint)
+				_, message, err := ws.ReadMessage()
+				if err != nil {
+					logErr("Misskey stream broken: ", endpoint)
+					return
+				}
+
+				// Print the message to the console.
+				logDebug("Ending Misskey Stream loop for ", endpoint)
+				var misskeyreply MisskeyReply
+				err = json.Unmarshal(message, &misskeyreply)
+				if err != nil {
+					logErr("Unable to parse data from "+endpoint+", but still connected, err: ", err)
+					break
+				}
+				//		newactivity := misskeyreply.Body.Body
+
+				var newactivity string
+
+				if misskeyreply.Body.Body.Uri != "" { // Remote Message
+					newactivity = misskeyreply.Body.Body.Uri
+					matchset := re.FindStringSubmatch(newactivity)
+					if matchset != nil {
+						newinstance := matchset[1]
+						logDebug("Checking new instance from Misskey Stream: ", newinstance)
+						go CheckInstance(newinstance, endpoint)
+					}
+				} else { // Local Message
+					newactivity = "https://" + endpoint + "/notes/" + misskeyreply.Body.Body.Id
+				}
+
+				logDebug("Misskey new URI ", newactivity, " from instance: ", endpoint)
+
+				go check_activity(newactivity)
+			}
+			// Close the WebSocket connection.
+			ws.Close()
+			time.Sleep(time.Minute * 10)
+		}
+	}
+}
+
+func StreamPleroma(endpoint string) {
+	wss_url := "wss://" + endpoint + "/api/v1/streaming/?stream=public"
+	var pleromaHeader shared.PleromaStreamHeader
+	var newactivity shared.ReportActivity
+	var err error
+
+	for {
+		var tries int
+		var ws *websocket.Conn
+		for tries = 0; tries < 10; tries++ {
+			ws, _, err = websocket.DefaultDialer.Dial(wss_url, nil)
+			if err != nil {
+				continue
+			}
+			break
+		}
+
+		if tries == 10 {
+			logWarn("Unable to connect to " + endpoint + " after 10 tries, exiting")
+			return
+		}
+
+		ri_mutex.Lock()
+		m := runninginstances[endpoint]
+		m.Status = shared.RUNNING
+		m.LastRun = "Streaming"
+		runninginstances[endpoint] = m
+		ri_mutex.Unlock()
+
+		for {
+			logDebug("Starting Pleroma Stream loop for ", endpoint)
+			_, p, err := ws.ReadMessage()
+			if err != nil {
+				logErr("Unable to read message from Pleroma stream: ", endpoint, " Err: ", err)
+				break
+			}
+
+			err = json.Unmarshal(p, &pleromaHeader)
+			if err != nil {
+				logErr("Unable to parse data from "+endpoint+", but still connected, err: ", err)
+				break
+			}
+
+			switch pleromaHeader.Event {
+			case "update":
+				err = json.Unmarshal([]byte(pleromaHeader.Payload), &newactivity)
+				if err != nil {
+					logErr("Unable to parse data from " + endpoint + ", but still connected.")
+					break
+				}
+				go check_activity(newactivity.Uri)
+				matchset := re.FindStringSubmatch(newactivity.Uri)
+				if matchset != nil {
+					newinstance := matchset[1]
+					logDebug("Checking new instance from Pleroma Stream: ", newinstance)
+					go CheckInstance(newinstance, endpoint)
+				}
+			default:
+				logDebug("Unimplemented pleroma stream activity: ", pleromaHeader.Event)
+				continue
+			}
+			logDebug("Ending Pleroma stream loop for ", endpoint)
+		}
+		ws.Close()
+		//		time.Sleep(time.Minute * 10)
+	}
+}
+
+func StreamMastodon(endpoint string, o *shared.RunningInstance) {
+	logDebug("Stream Mastodon for ", endpoint)
+	stream_client := BuildClient(endpoint)
+
+	var retry bool
+
+	api_timeline := "https://" + endpoint + "/api/v1/streaming/public"
+
+	for {
+		logDebug("StreamMastodon: " + endpoint + " URL: " + api_timeline)
+		req, err := http.NewRequest("GET", api_timeline, nil)
+		req.Header.Set("User-Agent", "Tusky")
+		if err != nil {
+			logFatal("Unable to create new request for " + endpoint + ", exiting.")
+			return
+		}
+
+		var resp *http.Response
+
+		for tries := 0; tries < 10; tries++ {
+			resp, err = stream_client.Do(req)
+			if err != nil {
+				time.Sleep(time.Minute * 5)
+				logWarn("Failure connecting to "+req.URL.Scheme+"://"+req.URL.Host+req.URL.Path+", attempt ", tries+1, ", sleeping for 5 minutes, ", err)
+				continue
+			}
+			break
+		}
+		if err != nil {
+			logErr("Unable to stream "+api_timeline+": ", err)
+			return
+		}
+		defer resp.Body.Close()
+
+		ri_mutex.Lock()
+		m := runninginstances[endpoint]
+		m.Status = shared.RUNNING
+		m.LastRun = "Streaming"
+		runninginstances[endpoint] = m
+		ri_mutex.Unlock()
+
+		s := bufio.NewScanner(resp.Body)
+		var name string
+		for s.Scan() {
+			logDebug("Starting Mastodon stream loop for ", endpoint)
+			line := s.Text()
+			token := strings.SplitN(line, ":", 2)
+			var newactivity shared.ReportActivity
+			if len(token) != 2 {
+				continue
+			}
+
+			switch strings.TrimSpace(token[0]) {
+			case "event":
+				name = strings.TrimSpace(token[1])
+				continue
+			case "data":
+				switch name {
+				case "update":
+					if len(token) >= 2 && len(token[1]) >= 2 {
+						continue
+					}
+					jsondata := token[1][1:]
+					err := json.Unmarshal([]byte(jsondata), &newactivity)
+					if err != nil {
+						logDebug("Unable to parse data from " + endpoint + ", but still connected.")
+						continue
+					}
+					retry = true
+				default:
+					continue
+				}
+			default:
+				continue
+			}
+
+			go check_activity(newactivity.Uri)
+
+			matchset := re.FindStringSubmatch(newactivity.Uri)
+			if matchset != nil {
+				newinstance := matchset[1]
+				logDebug("Checking new instance from Mastodon Stream: ", newinstance)
+				go CheckInstance(newinstance, endpoint)
+			}
+			logDebug("Ending Mastodon stream loop for ", endpoint)
+		}
+		if retry == true {
+			time.Sleep(time.Minute * 10)
+		} else {
+			break
+		}
+	}
+}

fedilogue/testhelper.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+// AssertEqual checks if values are equal
+func AssertEqual(t *testing.T, a interface{}, b interface{}) {
+	if a == b {
+		return
+	}
+	// debug.PrintStack()
+	t.Errorf("Received %v (type %v), expected %v (type %v)", a, reflect.TypeOf(a), b, reflect.TypeOf(b))
+}

fedilogue/web.go

@@ -0,0 +1,311 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+)
+
+// CreateObject - Used by post web receiver
+type CreateObject struct {
+	ID      string   `json:"id"`
+	Actor   string   `json:"actor"`
+	Cc      []string `json:"cc"`
+	Content string   `json:"content"`
+	To      []string `json:"to"`
+	Type    string   `json:"type"`
+}
+
+// CreateObject - Used by post web receiver
+type AnnounceObject struct {
+	ID     string   `json:"id"`
+	Actor  string   `json:"actor"`
+	To     []string `json:"to"`
+	Type   string   `json:"type"`
+	Object string   `json:"object"`
+}
+
+// RelayBase - The base object used by web receiver
+type RelayBase struct {
+	Actor     string          `json:"actor"`
+	Cc        []string        `json:"cc"`
+	Object    json.RawMessage `json:"Object"`
+	ID        string          `json:"id"`
+	Published string          `json:"published"`
+	To        []string        `json:"to"`
+	Type      string          `json:"type"`
+}
+
+func hostmeta(w http.ResponseWriter, r *http.Request) {
+	fmt.Println("PATH --> ", r.URL.Path)
+	host := r.Host
+	xml := "<?xml version=\"1.0\" encoding=\"UTF-8\"?><XRD xmlns=\"http://docs.oasis-open.org/ns/xri/xrd-1.0\"><Link rel=\"lrdd\" template=\"https://" + host + "/.well-known/webfinger?resource={uri}\" type=\"application/xrd+xml\" /></XRD>"
+	w.Header().Set("Content-Type", "application/xrd+xml")
+	fmt.Fprintf(w, xml)
+}
+
+func webfinger(w http.ResponseWriter, r *http.Request) {
+	fmt.Println("PATH --> ", r.URL.Path)
+	host := r.Host
+
+	webfingermap := make(map[string]interface{})
+	webfingermap["subject"] = "acct:fedilogue@" + host
+	webfingermap["aliases"] = []string{"https://" + host + "/users/fedilogue"}
+	link0 := make(map[string]string)
+	link0["rel"] = "http://webfinger.net/rel/profile-page"
+	link0["type"] = "text/html"
+	link0["href"] = "https://" + host + "/users/fedilogue"
+	link1 := make(map[string]string)
+	link1["rel"] = "self"
+	link1["type"] = "application/activity+json"
+	link1["href"] = "https://" + host + "/users/fedilogue"
+	link2 := make(map[string]string)
+	link2["rel"] = "self"
+	link2["type"] = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""
+	link2["href"] = "https://" + host + "/users/fedilogue"
+	link3 := make(map[string]string)
+	link3["rel"] = "http://ostatus.org/schema/1.0/subscribe"
+	link3["template"] = "https://" + host + "/ostatus_subscribe?acct={uri}"
+
+	links := []map[string]string{link0, link1, link2, link3}
+	webfingermap["links"] = links
+
+	webfingerbin, err := json.Marshal(webfingermap)
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	webfingerstr := string(webfingerbin)
+
+	query := r.URL.Query()
+	resourceRaw, exists := query["resource"]
+
+	if exists {
+		resource := resourceRaw[0]
+		if resource != "acct:fedilogue@"+host {
+			fmt.Println("Writes properly but wrong acct")
+			w.Header().Set("Content-Type", "application/json; charset=utf-8")
+			fmt.Fprintf(w, webfingerstr)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		fmt.Fprintf(w, webfingerstr)
+	} else {
+		fmt.Println(query)
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+}
+
+func inboxHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		defer r.Body.Close()
+
+		var findtype RelayBase
+		err = json.Unmarshal(body, &findtype)
+		if err != nil {
+			logWarn("Unable to unmarshal here, exiting...")
+			return
+		}
+
+		switch findtype.Type {
+		case "Create":
+			var createobject CreateObject
+
+			err = json.Unmarshal(body, &createobject)
+			if err != nil {
+				return
+			}
+
+			go check_activity(createobject.ID)
+			slashend := strings.Index(createobject.ID[8:], "/")
+			newinstance := createobject.ID[8 : 8+slashend]
+
+			go CheckInstance(newinstance, "")
+
+		case "Update":
+		case "Reject":
+		case "Add":
+		case "Remove":
+		case "Follow":
+		case "Accept":
+		case "Like":
+		case "Announce":
+			var announceobject AnnounceObject
+			err = json.Unmarshal(body, &announceobject)
+			if err != nil {
+				fmt.Println(err.Error())
+				return
+			}
+			check_activity(announceobject.Object)
+
+			matchset := re.FindStringSubmatch(announceobject.Object)
+			if matchset != nil {
+				newinstance := matchset[1]
+				go CheckInstance(newinstance, "")
+			}
+		case "Delete":
+		case "Undo":
+		default:
+			logWarn("Unknown ActivityPub request:", findtype.Type)
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintf(w, "{}")
+	}
+}
+
+func internalFetch(w http.ResponseWriter, r *http.Request) {
+
+	publicKeyBytes, err := ioutil.ReadFile("keys/public.pem")
+	publicKeyString := string(publicKeyBytes)
+	publicKeyString = strings.Replace(publicKeyString, "\n", "\\n", -11)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	staticjson := fmt.Sprintf(`{"@context":["https://www.w3.org/ns/activitystreams","https://%[1]s/schemas/litepub-0.1.jsonld",{"@language":"und"}],"endpoints":{"oauthAuthorizationEndpoint":"https://%[1]s/oauth/authorize","oauthRegistrationEndpoint":"https://%[1]s/api/v1/apps","oauthTokenEndpoint":"https://%[1]s/oauth/token","sharedInbox":"https://%[1]s/inbox","uploadMedia":"https://%[1]s/api/ap/upload_media"},"followers":"https://%[1]s/internal/fetch/followers","following":"https://%[1]s/internal/fetch/following","id":"https://%[1]s/internal/fetch","inbox":"https://%[1]s/internal/fetch/inbox","invisible":true,"manuallyApprovesFollowers":false,"name":"Pleroma","preferredUsername":"fedilogue","publicKey":{"id":"https://%[1]s/internal/fetch#main-key","owner":"https://%[1]s/internal/fetch","publicKeyPem":"%[2]s"},"summary":"Fedilogue Key or something.","type":"Application","url":"https://%[1]s/internal/fetch"}`, r.Host, publicKeyString)
+
+	fmt.Fprintf(w, staticjson)
+
+}
+
+func relay(w http.ResponseWriter, r *http.Request) {
+
+	fmt.Println("Someone out there requested /relay")
+
+	publicKeyBytes, err := ioutil.ReadFile("keys/public.pem")
+	publicKeyString := string(publicKeyBytes)
+	publicKeyString = strings.Replace(publicKeyString, "\n", "\\n", -11)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	staticjson := fmt.Sprintf(`{"@context":["https://www.w3.org/ns/activitystreams","https://%[1]s/schemas/litepub-0.1.jsonld",{"@language":"und"}],"alsoKnownAs":[],"attachment":[],"capabilities":{},"discoverable":false,"endpoints":{"oauthAuthorizationEndpoint":"https://%[1]s/oauth/authorize","oauthRegistrationEndpoint":"https://%[1]s/api/v1/apps","oauthTokenEndpoint":"https://%[1]s/oauth/token","sharedInbox":"https://%[1]s/inbox","uploadMedia":"https://%[1]s/api/ap/upload_media"},"featured":"https://%[1]s/relay/collections/featured","followers":"https://%[1]s/relay/followers","following":"https://%[1]s/relay/following","id":"https://%[1]s/relay","inbox":"https://%[1]s/relay/inbox","manuallyApprovesFollowers":false,"name":null,"outbox":"https://%[1]s/relay/outbox","preferredUsername":"relay","publicKey":{"id":"https://%[1]s/relay#main-key","owner":"https://%[1]s/relay","publicKeyPem":"%[2]s"},"summary":"","tag":[],"type":"Person","url":"https://%[1]s/relay"}`, r.Host, publicKeyString)
+
+	fmt.Fprintf(w, staticjson)
+
+}
+
+func usersFedilogueFollowers(w http.ResponseWriter, r *http.Request) {
+	fmt.Println("PATH --> ", r.URL.Path)
+	host := r.Host
+	contextlist := map[string]string{"@language": "und"}
+
+	context := []interface{}{"https://www.w3.org/ns/activitystreams", "https://" + host + "/schemas/litepub-0.1.jsonld", contextlist}
+	followersmap := make(map[string]interface{})
+	followersmap["@context"] = context
+
+	staticjson := "{\"@context\":[\"https://www.w3.org/ns/activitystreams\",\"https://" + host + "/schemas/litepub-0.1.jsonld\",{\"@language\":\"und\"}],\"first\":{\"id\":\"https://" + host + "/users/fedilogue/followers?page=1\",\"next\":\"https://" + host + "/users/fedilogue/followers?page=2\",\"orderedItems\":[\"https://mastodon.host/users/federationbot\"],\"partOf\":\"https://" + host + "/users/fedilogue/followers\",\"totalItems\":1,\"type\":\"OrderedCollectionPage\"},\"id\":\"https://" + host + "/users/fedilogue/followers\",\"totalItems\":1,\"type\":\"OrderedCollection\"}"
+
+	w.Header().Set("Content-Type", "application/activity+json; charset=utf-8")
+	fmt.Fprintf(w, staticjson)
+}
+
+func usersFedilogueFollowing(w http.ResponseWriter, r *http.Request) {
+	host := r.Host
+	fmt.Println("PATH --> ", r.URL.Path)
+	staticjson := "{\"@context\": [\"https://www.w3.org/ns/activitystreams\", \"https://" + host + "/schemas/litepub-0.1.jsonld\", {\"@language\": \"und\"}], \"first\": {\"id\": \"https://" + host + "/users/fedilogue/following?page=1\", \"orderedItems\": [], \"partOf\": \"https://" + host + "/users/fedilogue/following\", \"totalItems\": 0, \"type\": \"OrderedCollectionPage\"}, \"id\": \"https://" + host + "/users/fedilogue/following\", \"totalItems\": 0, \"type\": \"OrderedCollection\"}"
+	w.Header().Set("Content-Type", "application/activity+json; charset=utf-8")
+	fmt.Fprintf(w, staticjson)
+}
+
+func usersFedilogue(w http.ResponseWriter, r *http.Request) {
+	fmt.Println("PATH --> ", r.URL.Path)
+	host := r.Host
+
+	fmt.Println(r.Host)
+	fmt.Println(r.Header["Accept"])
+
+	publickeybin, err := ioutil.ReadFile("keys/public.pem")
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	publickeypemstr := string(publickeybin)
+
+	publicKey := map[string]string{"id": "https://" + host + "/users/fedilogue#main-key", "owner": "https://" + host + "/users/fedilogue", "publicKeyPem": publickeypemstr}
+
+	capabilities := map[string]bool{}
+	tag := []string{}
+	contextlist := map[string]string{"@language": "und"}
+	attachment := []string{}
+	endpoints := map[string]string{"oauthAuthorizationEndpoint": "https://" + host + "/oauth/authorize", "oauthRegistrationEndpoint": "https://" + host + "/api/v1/apps", "oauthTokenEndpoint": "https://" + host + "/oauth/token", "sharedInbox": "https://" + host + "/inbox", "uploadMedia": "https://" + host + "/api/ap/upload_media"}
+
+	context := []interface{}{"https://www.w3.org/ns/activitystreams", "https://" + host + "/schemas/litepub-0.1.jsonld", contextlist}
+	actorjsonmap := make(map[string]interface{})
+	actorjsonmap["@context"] = context
+	actorjsonmap["attachment"] = attachment
+	actorjsonmap["capabilities"] = capabilities
+	actorjsonmap["discoverable"] = false
+	actorjsonmap["endpoints"] = endpoints
+	actorjsonmap["followers"] = "https://" + host + "/users/fedilogue/followers"
+	actorjsonmap["following"] = "https://" + host + "/users/fedilogue/following"
+	actorjsonmap["id"] = "https://" + host + "/users/fedilogue"
+	actorjsonmap["inbox"] = "https://" + host + "/users/fedilogue/inbox"
+	actorjsonmap["manuallyApprovesFollowers"] = false
+	actorjsonmap["name"] = "Fedilogue Mass Follower"
+	actorjsonmap["outbox"] = "https://" + host + "/users/fedilogue/outbox"
+	actorjsonmap["preferredUsername"] = "fedilogue"
+	actorjsonmap["publicKey"] = publicKey
+	actorjsonmap["summary"] = ""
+	actorjsonmap["tag"] = tag
+	actorjsonmap["type"] = "Application"
+	actorjsonmap["uri"] = "https://" + host + "/users/fedilogue"
+
+	actorjsonbin, err := json.Marshal(actorjsonmap)
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	actorjsonstr := string(actorjsonbin)
+	w.Header().Set("Content-Type", "application/activity+json; charset=utf-8")
+	fmt.Fprintf(w, actorjsonstr)
+}
+
+func errorHandler(w http.ResponseWriter, r *http.Request) {
+	for name, headers := range r.Header {
+		fmt.Println("ROW: ", name, " ", headers)
+		for _, h := range headers {
+			fmt.Fprintf(w, "%v: %v\n", name, h)
+		}
+	}
+
+	reqBody, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("POST DATA: %s\n", reqBody)
+
+}
+
+func webmain() {
+
+	webreceiver := http.NewServeMux()
+
+	webreceiver.HandleFunc("/.well-known/webfinger", webfinger)
+	webreceiver.HandleFunc("/.well-known/host-meta", hostmeta)
+	webreceiver.HandleFunc("/inbox", inboxHandler())
+	webreceiver.HandleFunc("/internal/fetch", internalFetch)
+	webreceiver.HandleFunc("/relay", relay)
+	webreceiver.HandleFunc("/users/fedilogue", usersFedilogue)
+	webreceiver.HandleFunc("/users/fedilogue/followers", usersFedilogueFollowers)
+	webreceiver.HandleFunc("/users/fedilogue/following", usersFedilogueFollowing)
+	webreceiver.HandleFunc("/", errorHandler)
+	log.Print("Starting HTTP inbox on port 127.0.0.1:8042")
+	log.Fatal(http.ListenAndServe("127.0.0.1:8042", webreceiver))
+}