diff --git a/poll/engine.go b/poll/engine.go index 874ebd1..21d4de9 100644 --- a/poll/engine.go +++ b/poll/engine.go @@ -43,8 +43,8 @@ type ReportPost struct { Created_at string `json:"created_at"` // Derived values - StrippedContent string - Posthash []byte + normalized string + posthash []byte } type AccountType struct { @@ -160,13 +160,12 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor // Calculate the post hash fmt.Fprint(posthash, newpost.Url) - fmt.Fprint(posthash, newpost.StrippedContent) + fmt.Fprint(posthash, newpost.normalized) fmt.Fprint(posthash, newpost.Account.Acct) fmt.Fprint(posthash, newpost.Account.Display_name) - fmt.Fprint(posthash, newpost.Account.Url) - newpost.Posthash = posthash.Sum(nil) + newpost.posthash = posthash.Sum(nil) - newpost.StrippedContent = p.Sanitize(newpost.Content) + newpost.normalized = strings.ToLower(p.Sanitize(newpost.Content)) reportPostChan <- newpost @@ -234,11 +233,11 @@ func DeferPollRun(pollmessage PollMessage, runninginstances *[]RunningInstance, } } - delay := 2 + delay := 10 if pollmessage.status == 200 && pollmessage.numposts <= 10 { - delay = 2 + delay = 10 } else if pollmessage.status == 200 && pollmessage.numposts > 10 { - delay = 5 + delay = 15 } else if pollmessage.status == 429 { delay = 30 } else { @@ -309,7 +308,7 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) { } // Insert new post if new - _, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, strippedcontent, acct_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.StrippedContent, acctid, reportpost.Posthash) + _, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, normalized, acct_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, acctid, reportpost.posthash) if err != nil { // For now I want to know why this failed. fmt.Println("Second ", err) os.Exit(1) // For now I want this to die and learn why it failed diff --git a/poll/tables.sql b/poll/tables.sql index b88a3e1..9a8a1bf 100644 --- a/poll/tables.sql +++ b/poll/tables.sql @@ -1,22 +1,23 @@ CREATE TABLE accounts ( - id serial NOT NULL PRIMARY KEY, - acct VARCHAR(100) NOT NULL UNIQUE, + id serial NOT NULL PRIMARY KEY, + + acct VARCHAR(1000) NOT NULL UNIQUE, avatar VARCHAR(2083) NOT NULL, bot boolean, - created_at VARCHAR(100) NOT NULL, - display_name VARCHAR(100) NOT NULL, + created_at timestamptz DEFAULT NOW(), + display_name VARCHAR(2000) NOT NULL, url VARCHAR(2083) NOT NULL ); CREATE TABLE posts ( id serial NOT NULL PRIMARY KEY, + url VARCHAR(2083) NOT NULL, content text, - strippedcontent text, created_at timestamptz DEFAULT NOW(), + normalized text, acct_id int NOT NULL REFERENCES accounts (id), - posthash bytea UNIQUE );