farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated tables, hashing and polling time

Browse Source
This commit is contained in:
farhan 2020-10-29 15:06:45 +00:00
parent 5ce495b81c
commit 9c2f2c8a0d
2 changed files with 16 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
poll/engine.go
View File

@ -43,8 +43,8 @@ type ReportPost struct {
Created_at string `json:"created_at"` Created_at string `json:"created_at"`
// Derived values // Derived values
StrippedContent string normalized string
Posthash []byte posthash []byte
} }
type AccountType struct { type AccountType struct {
@ -160,13 +160,12 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
// Calculate the post hash // Calculate the post hash
fmt.Fprint(posthash, newpost.Url) fmt.Fprint(posthash, newpost.Url)
fmt.Fprint(posthash, newpost.StrippedContent) fmt.Fprint(posthash, newpost.normalized)
fmt.Fprint(posthash, newpost.Account.Acct) fmt.Fprint(posthash, newpost.Account.Acct)
fmt.Fprint(posthash, newpost.Account.Display_name) fmt.Fprint(posthash, newpost.Account.Display_name)
fmt.Fprint(posthash, newpost.Account.Url) newpost.posthash = posthash.Sum(nil)
newpost.Posthash = posthash.Sum(nil)
newpost.StrippedContent = p.Sanitize(newpost.Content) newpost.normalized = strings.ToLower(p.Sanitize(newpost.Content))
reportPostChan <- newpost reportPostChan <- newpost
@ -234,11 +233,11 @@ func DeferPollRun(pollmessage PollMessage, runninginstances *[]RunningInstance,
} }
} }
delay := 2 delay := 10
if pollmessage.status == 200 && pollmessage.numposts <= 10 { if pollmessage.status == 200 && pollmessage.numposts <= 10 {
delay = 2 delay = 10
} else if pollmessage.status == 200 && pollmessage.numposts > 10 { } else if pollmessage.status == 200 && pollmessage.numposts > 10 {
delay = 5 delay = 15
} else if pollmessage.status == 429 { } else if pollmessage.status == 429 {
delay = 30 delay = 30
} else { } else {
@ -309,7 +308,7 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
} }
// Insert new post if new // Insert new post if new
_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, strippedcontent, acct_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.StrippedContent, acctid, reportpost.Posthash) _, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, normalized, acct_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, acctid, reportpost.posthash)
if err != nil { // For now I want to know why this failed. if err != nil { // For now I want to know why this failed.
fmt.Println("Second ", err) fmt.Println("Second ", err)
os.Exit(1) // For now I want this to die and learn why it failed os.Exit(1) // For now I want this to die and learn why it failed

11
poll/tables.sql
View File

@ -1,22 +1,23 @@
CREATE TABLE accounts ( CREATE TABLE accounts (
id serial NOT NULL PRIMARY KEY, id serial NOT NULL PRIMARY KEY,
acct VARCHAR(100) NOT NULL UNIQUE,
acct VARCHAR(1000) NOT NULL UNIQUE,
avatar VARCHAR(2083) NOT NULL, avatar VARCHAR(2083) NOT NULL,
bot boolean, bot boolean,
created_at VARCHAR(100) NOT NULL, created_at timestamptz DEFAULT NOW(),
display_name VARCHAR(100) NOT NULL, display_name VARCHAR(2000) NOT NULL,
url VARCHAR(2083) NOT NULL url VARCHAR(2083) NOT NULL
); );
CREATE TABLE posts ( CREATE TABLE posts (
id serial NOT NULL PRIMARY KEY, id serial NOT NULL PRIMARY KEY,
url VARCHAR(2083) NOT NULL, url VARCHAR(2083) NOT NULL,
content text, content text,
strippedcontent text,
created_at timestamptz DEFAULT NOW(), created_at timestamptz DEFAULT NOW(),
normalized text,
acct_id int NOT NULL REFERENCES accounts (id), acct_id int NOT NULL REFERENCES accounts (id),
posthash bytea UNIQUE posthash bytea UNIQUE
); );