farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

commits to the database, added accounts and created_time

Browse Source
This commit is contained in:
farhan 2020-10-26 02:48:52 +00:00
parent f460900f41
commit 84a07edf07
2 changed files with 23 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
poll/engine.go
View File

@ -32,6 +32,7 @@ type ReportPost struct {
Url string `json:"url"`
Account AccountType
Content string `json:"content"`
Created_at string `json:"created_at"`
// Derived values
StrippedContent string
@ -87,9 +88,10 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
p := bluemonday.NewPolicy()
newposts := make([]ReportPost, 0)
api_timeline := "https://" + endpoint + "/api/v1/timelines/public?min_id=" + min_id
api_timeline := "https://" + endpoint + "/api/v1/timelines/public?since_id=40&min_id=" + min_id
resp, err := http.Get(api_timeline)
if err != nil {
fmt.Println("Failure to retrieve HTTPS data...")
log.Fatal(err)
pollMessageChan <- PollMessage{endpoint, resp.StatusCode, "" , 0}
}
@ -97,9 +99,11 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
body, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &newposts)
if err != nil {
pollMessageChan <- PollMessage{endpoint, resp.StatusCode, "", 0}
fmt.Println("Failure to unmarshal 1")
fmt.Println(string(body))
log.Fatal(err)
panic(err)
pollMessageChan <- PollMessage{endpoint, resp.StatusCode, "", 0}
}
numposts := 0
@ -113,7 +117,6 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
fmt.Fprint(posthash, newpost.Url)
fmt.Fprint(posthash, newpost.Content)
fmt.Fprint(posthash, newpost.Account.Acct)
fmt.Fprint(posthash, newpost.Account.Created_at)
fmt.Fprint(posthash, newpost.Account.Display_name)
fmt.Fprint(posthash, newpost.Account.Url)
newpost.Posthash = posthash.Sum(nil)
@ -195,13 +198,12 @@ func NewInstance(endpoint string, runninginstances *[]RunningInstance, reportIns
*runninginstances = append(*runninginstances, newinstance)
go StartInstancePoll(endpoint, "", reportPostChan, pollMessageChan)
fmt.Println("Temporarily disabled Peer Hunting")
// fmt.Println("Temporarily disabled Peer Hunting")
// go StartGetPeers(endpoint, reportInstanceChan)
}
func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
fmt.Println("Writing post", reportpost)
// sqlWritePost := `INSERT INTO post (url, content, strippedcontent, posthash) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (posthash) DO NOTHING`
// fmt.Println("Writing post", reportpost)
conn, err := pool.Acquire(context.Background())
if err != nil {
@ -210,12 +212,23 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
}
defer conn.Release()
_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, strippedcontent, posthash) VALUES ($1, $2, $3, $4) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.StrippedContent, reportpost.Posthash)
// Insert new post if new
_, err = conn.Exec(context.Background(), "INSERT INTO posts (url, content, created_at, strippedcontent, posthash) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (posthash) DO NOTHING", reportpost.Url, reportpost.Content, reportpost.Created_at, reportpost.StrippedContent, reportpost.Posthash)
if err != nil {
fmt.Println("Error on channel???")
fmt.Println(err)
os.Exit(1)
}
// Insert new account if new
_, err = conn.Exec(context.Background(), "INSERT INTO accounts (acct, avatar, bot, created_at, display_name, url) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (acct) DO NOTHING", reportpost.Account.Acct, reportpost.Account.Avatar, reportpost.Account.Bot, reportpost.Account.Created_at, reportpost.Account.Display_name, reportpost.Account.Url)
if err != nil {
fmt.Println("Error on channel???")
fmt.Println(err)
os.Exit(1)
}
// fmt.Println(reportpost.Account)
}
func main() {

4
poll/tables.psql
View File

@ -1,6 +1,6 @@
CREATE TABLE accounts (
id serial NOT NULL PRIMARY KEY,
acct VARCHAR(100) NOT NULL,
acct VARCHAR(100) NOT NULL UNIQUE,
avatar VARCHAR(2083) NOT NULL,
bot boolean,
created_at VARCHAR(100) NOT NULL,
@ -13,6 +13,8 @@ CREATE TABLE posts (
url VARCHAR(2083) NOT NULL,
content text,
strippedcontent text,
created_at timestamptz DEFAULT NOW(),
posthash bytea UNIQUE
);