farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

renamed column name, removed html encoding in normalization field

Browse Source
This commit is contained in:
farhan 2020-10-29 23:49:38 +00:00
parent 97a8cbf578
commit 4222cb97e0
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
poll/engine.go
View File

@ -9,6 +9,7 @@ import (
"net/http" "net/http"
"context" "context"
"strings" "strings"
"html"
"time" "time"
"fmt" "fmt"
"net" "net"
@ -165,7 +166,7 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
fmt.Fprint(posthash, newpost.Account.Display_name) fmt.Fprint(posthash, newpost.Account.Display_name)
newpost.posthash = posthash.Sum(nil) newpost.posthash = posthash.Sum(nil)
newpost.normalized = strings.ToLower(p.Sanitize(newpost.Content)) newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content)))
reportPostChan <- newpost reportPostChan <- newpost
@ -298,17 +299,16 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
defer conn.Release() defer conn.Release()
// Insert new account if new // Insert new account if new
var acctid int var accountid int
err = conn.QueryRow(context.Background(), "INSERT INTO accounts (acct, avatar, bot, created_at, display_name, url) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT(acct) DO UPDATE SET acct=EXCLUDED.acct RETURNING id", reportpost.Account.Acct, reportpost.Account.Avatar, reportpost.Account.Bot, reportpost.Account.Created_at, reportpost.Account.Display_name, reportpost.Account.Url).Scan(&acctid) err = conn.QueryRow(context.Background(), "INSERT INTO accounts (acct, avatar, bot, created_at, display_name, url) VALUES($1, $2, $3, $4, $5, $6) ON CONFLICT(acct) DO UPDATE SET acct=EXCLUDED.acct RETURNING id", reportpost.Account.Acct, reportpost.Account.Avatar, reportpost.Account.Bot, reportpost.Account.Created_at, reportpost.Account.Display_name, reportpost.Account.Url).Scan(&accountid)
if err != nil { if err != nil {
fmt.Println("First ", err) fmt.Println("First ", err)
fmt.Println(reportpost.Account.Acct, reportpost.Account.Avatar, reportpost.Account.Bot, reportpost.Account.Created_at, reportpost.Account.Display_name, reportpost.Account.Url)
os.Exit(1) // For now I want this to die and learn why it failed os.Exit(1) // For now I want this to die and learn why it failed
return return
} }
// Insert new post if new // Insert new post if new
_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, normalized, acct_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, acctid, reportpost.posthash) _, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, normalized, account_id, posthash) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.normalized, accountid, reportpost.posthash)
if err != nil { // For now I want to know why this failed. if err != nil { // For now I want to know why this failed.
fmt.Println("Second ", err) fmt.Println("Second ", err)
os.Exit(1) // For now I want this to die and learn why it failed os.Exit(1) // For now I want this to die and learn why it failed

2
poll/tables.sql
View File

@ -17,7 +17,7 @@ CREATE TABLE posts (
created_at timestamptz DEFAULT NOW(), created_at timestamptz DEFAULT NOW(),
normalized text, normalized text,
acct_id int NOT NULL REFERENCES accounts (id), account_id int NOT NULL REFERENCES accounts (id),
posthash bytea UNIQUE posthash bytea UNIQUE
); );