updated tables, hashing and polling time

poll/engine.go

@@ -43,8 +43,8 @@ type ReportPost struct {
 	Created_at		string `json:"created_at"`
 
 	// Derived values
-	StrippedContent			string
-	Posthash			[]byte
+	normalized			string
+	posthash			[]byte
 }
 
 type AccountType struct {
@@ -160,13 +160,12 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
 
 		// Calculate the post hash
 		fmt.Fprint(posthash, newpost.Url)
-		fmt.Fprint(posthash, newpost.StrippedContent)
+		fmt.Fprint(posthash, newpost.normalized)
 		fmt.Fprint(posthash, newpost.Account.Acct)
 		fmt.Fprint(posthash, newpost.Account.Display_name)
-		fmt.Fprint(posthash, newpost.Account.Url)
-		newpost.Posthash = posthash.Sum(nil)
+		newpost.posthash = posthash.Sum(nil)
 
-		newpost.StrippedContent = p.Sanitize(newpost.Content)
+		newpost.normalized = strings.ToLower(p.Sanitize(newpost.Content))
 
 		reportPostChan <- newpost
 
@@ -234,11 +233,11 @@ func DeferPollRun(pollmessage PollMessage, runninginstances *[]RunningInstance,
 		}
 	}
 
-	delay := 2
+	delay := 10
 	if pollmessage.status == 200 && pollmessage.numposts <= 10 {
-		delay = 2
+		delay = 10
 	} else if pollmessage.status == 200 && pollmessage.numposts > 10 {
-		delay = 5
+		delay = 15
 	} else if pollmessage.status == 429 {
 		delay = 30
 	} else {
@@ -309,7 +308,7 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
 	}
 
 	// Insert new post if new
-	_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, strippedcontent, acct_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.StrippedContent, acctid, reportpost.Posthash)
+	_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, normalized, acct_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, acctid, reportpost.posthash)
 	if err != nil { // For now I want to know why this failed.
 		fmt.Println("Second ", err)
 		os.Exit(1) // For now I want this to die and learn why it failed

poll/tables.sql

@@ -1,22 +1,23 @@
 CREATE TABLE accounts (
-	id serial	NOT NULL PRIMARY KEY,
-	acct		VARCHAR(100) NOT NULL UNIQUE,
+	id 		serial NOT NULL PRIMARY KEY,
+
+	acct		VARCHAR(1000) NOT NULL UNIQUE,
 	avatar		VARCHAR(2083) NOT NULL,
 	bot		boolean,
-	created_at	VARCHAR(100) NOT NULL,
-	display_name	VARCHAR(100) NOT NULL,
+	created_at	timestamptz DEFAULT NOW(),
+	display_name	VARCHAR(2000) NOT NULL,
 	url		VARCHAR(2083) NOT NULL
 );
 
 CREATE TABLE posts (
 	id		serial NOT NULL PRIMARY KEY,
+
 	url		VARCHAR(2083) NOT NULL,
 	content		text,
-	strippedcontent	text,
 	created_at	timestamptz DEFAULT NOW(),
 
+	normalized	text,
 	acct_id		int NOT NULL REFERENCES accounts (id),
-
 	posthash	bytea UNIQUE
 );