farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Capturinghashtag

Browse Source
This commit is contained in:
Fikrān Mutasā'il 2021-12-15 21:02:11 +00:00
parent 6befc66275
commit 2b697c3857
2 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
fedilogue/retrieve.go
View File

@ -3,7 +3,6 @@ package main
import ( import (
"context" "context"
"encoding/json" "encoding/json"
//"errors"
"html" "html"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -46,6 +45,12 @@ type ActorJson struct {
instance string instance string
} }
type TagType struct {
Type string `json:"type"`
Name string `json:"name"`
}
type PostJson struct { type PostJson struct {
id int id int
uri string `json:"id"` uri string `json:"id"`
@ -59,7 +64,7 @@ type PostJson struct {
Published time.Time `json:"published"` Published time.Time `json:"published"`
Source string `json:"source"` Source string `json:"source"`
Summary string `json:"summary"` Summary string `json:"summary"`
// Ignoring tag for now Tag []TagType `json:"tag"`
To []string `json:"to"` To []string `json:"to"`
Type string `json:"type"` Type string `json:"type"`
@ -144,7 +149,13 @@ func check_activity(uri string) {
activityjson.normalized = matchurl.ReplaceAllString(activityjson.normalized, "") activityjson.normalized = matchurl.ReplaceAllString(activityjson.normalized, "")
activityjson.normalized = spaceReg.ReplaceAllString(activityjson.normalized, " ") activityjson.normalized = spaceReg.ReplaceAllString(activityjson.normalized, " ")
_, err = pool.Exec(context.Background(), "INSERT INTO activities (document, normalized, instance) VALUES($1, $2, $3)", jsondocument, activityjson.normalized, activityjson.instance) var hashtags []string
for _, tag := range activityjson.Tag {
if tag.Type == "Hashtag" {
hashtags = append(hashtags, strings.ToLower(tag.Name))
}
}
_, err = pool.Exec(context.Background(), "INSERT INTO activities (document, normalized, instance, hashtags) VALUES($1, $2, $3, $4)", jsondocument, activityjson.normalized, activityjson.instance, hashtags)
if err != nil { if err != nil {
logWarn("Error inserting %s into `activities`: "+ uri, err) logWarn("Error inserting %s into `activities`: "+ uri, err)
return return

4
fedilogue/tables.sql
View File

@ -10,7 +10,8 @@ CREATE TABLE IF NOT EXISTS activities (
document JSONB, document JSONB,
normalized TEXT, normalized TEXT,
identifiedat TIMESTAMP with time zone DEFAULT now(), identifiedat TIMESTAMP with time zone DEFAULT now(),
instance VARCHAR(1000) NOT NULL instance VARCHAR(1000) NOT NULL,
hashtags VARCHAR(140)[]
); );
@ -33,6 +34,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS activities_uri_idx ON activities ( (document-
CREATE INDEX IF NOT EXISTS activities_published_idx ON activities ( (document->>'published') ); CREATE INDEX IF NOT EXISTS activities_published_idx ON activities ( (document->>'published') );
CREATE INDEX IF NOT EXISTS activities_identifiedat_idx ON activities (identifiedat); CREATE INDEX IF NOT EXISTS activities_identifiedat_idx ON activities (identifiedat);
CREATE INDEX IF NOT EXISTS hashtags_idx ON activities(hashtags);
CREATE INDEX IF NOT EXISTS normalized_idx ON activities USING gin(normalized_tsvector); CREATE INDEX IF NOT EXISTS normalized_idx ON activities USING gin(normalized_tsvector);