Compare commits

...

6 Commits

Author SHA1 Message Date
097c05b478 Woops! 2025-01-27 04:31:20 +00:00
808bac398a Removing accidental extra character 2025-01-25 01:17:34 +00:00
9a1b2c98a3 Updating due to upstream library update 2025-01-25 01:16:33 +00:00
d3efb1c8dd Updating postgres variables 2025-01-24 21:30:25 +00:00
c6ba6fdfde Updated Dockerfile
Accidentally committed a directory twice
2025-01-24 21:12:19 +00:00
22646f052d Deleting entrypoint.sh script 2025-01-24 04:49:34 +00:00
27 changed files with 47 additions and 761 deletions

View File

@ -1,6 +1,11 @@
services: services:
fedilogue: fedilogue:
build: fedilogger/ build: fedilogue/
environment:
POSTGRES_USER: $POSTGRES_USER
POSTGRES_PASSWORD: $POSTGRES_PASSWORD
POSTGRES_HOSTNAME: $POSTGRES_HOSTNAME
POSTGRES_DB: $POSTGRES_DB
depends_on: depends_on:
- db - db
ports: ports:
@ -10,16 +15,17 @@ services:
ports: ports:
- "127.0.0.1:6431:6432" - "127.0.0.1:6431:6432"
environment: environment:
POSTGRES_USER: $PGUSER POSTGRES_USER: $POSTGRES_USER
POSTGRES_PASSWORD: $PGPASS POSTGRES_PASSWORD: $POSTGRES_PASSWORD
POSTGRES_DB: $PGDATABASE POSTGRES_HOSTNAME: $POSTGRES_HOSTNAME
POSTGRES_DB: $POSTGRES_DB
depends_on: depends_on:
- db - db
db: db:
image: postgres:alpine image: postgres:alpine
environment: environment:
POSTGRES_USER: $PGUSER POSTGRES_USER: $POSTGRES_USER
POSTGRES_PASSWORD: $PGPASS POSTGRES_PASSWORD: $POSTGRES_PASSWORD
POSTGRES_DB: $PGDATABASE POSTGRES_DB: $POSTGRES_DB
volumes: volumes:
- ./postgres-data:/var/lib/postgresql - ./postgres-data:/var/lib/postgresql

View File

@ -1,11 +0,0 @@
FROM golang:latest AS build
WORKDIR /build
COPY go.mod go.sum .
RUN go mod download
COPY . .
RUN go build .
FROM alpine:latest
WORKDIR /app
COPY --from=build /build/fedilogger /app/fedilogger
ENTRYPOINT ["/app/restapi"]

View File

@ -1,731 +0,0 @@
CREATE TABLE IF NOT EXISTS actors (
id SERIAL PRIMARY KEY,
document JSONB,
identifiedat TIMESTAMP with time zone DEFAULT now(),
instance VARCHAR(1000) NOT NULL,
bot BOOLEAN DEFAULT FALSE
);
CREATE TABLE IF NOT EXISTS activities (
id SERIAL PRIMARY KEY,
document JSONB,
normalized TEXT,
identifiedat TIMESTAMP with time zone DEFAULT now(),
instance VARCHAR(1000) NOT NULL,
hashtags VARCHAR(140)[],
bot BOOLEAN DEFAULT FALSE
);
CREATE TABLE IF NOT EXISTS instances (
endpoint VARCHAR(2083) NOT NULL PRIMARY KEY UNIQUE,
state VARCHAR(16),
username VARCHAR(32),
password VARCHAR(32),
software VARCHAR(50),
banned BOOLEAN DEFAULT FALSE,
alwaysbot BOOLEAN DEFAULT FALSE
);
-- Autostart mastodon.social
INSERT INTO instances (endpoint) VALUES('mastodon.social');
-- Banned Instances
INSERT INTO instances (endpoint, banned) VALUES
('switter.at', true),
('xxxtumblr.org', true),
('sinblr.com', true),
('twitiverse.com', true),
('my.dirtyhobby.xyz', true),
('bae.st', true);
-- Alwaysbot instances
INSERT INTO instances (endpoint, alwaysbot) VALUES
('mstdn.foxfam.club', true),
('botsin.space', true),
('newsbots.eu', true);
ALTER TABLE activities
ADD normalized_tsvector tsvector
GENERATED ALWAYS AS (to_tsvector('english', normalized)) STORED;
CREATE UNIQUE INDEX IF NOT EXISTS actors_uri_idx ON actors ( (document->>'id') );
CREATE UNIQUE INDEX IF NOT EXISTS activities_uri_idx ON activities ( (document->>'id') );
CREATE INDEX IF NOT EXISTS activities_published_idx ON activities ( (document->>'published') );
CREATE INDEX IF NOT EXISTS activities_identifiedat_idx ON activities (identifiedat);
CREATE INDEX IF NOT EXISTS hashtags_idx ON activities(hashtags);
CREATE INDEX IF NOT EXISTS normalized_idx ON activities USING gin(normalized_tsvector);
CREATE INDEX IF NOT EXISTS actors_id_idx ON actors (id);
CREATE INDEX IF NOT EXISTS activities_id_idx ON activities (id);
CREATE TABLE IF NOT EXISTS stopwords (
word VARCHAR(20)
);
INSERT INTO stopwords (word)
VALUES
('a'),
('able'),
('about'),
('above'),
('abst'),
('accordance'),
('according'),
('accordingly'),
('across'),
('act'),
('actually'),
('added'),
('adj'),
('affected'),
('affecting'),
('affects'),
('after'),
('afterwards'),
('again'),
('against'),
('ah'),
('all'),
('almost'),
('alone'),
('along'),
('already'),
('also'),
('although'),
('always'),
('am'),
('among'),
('amongst'),
('an'),
('and'),
('announce'),
('another'),
('any'),
('anybody'),
('anyhow'),
('anymore'),
('anyone'),
('anything'),
('anyway'),
('anyways'),
('anywhere'),
('apparently'),
('approximately'),
('are'),
('aren'),
('arent'),
('arise'),
('around'),
('as'),
('aside'),
('ask'),
('asking'),
('at'),
('auth'),
('available'),
('away'),
('awfully'),
('b'),
('back'),
('be'),
('became'),
('because'),
('become'),
('becomes'),
('becoming'),
('been'),
('before'),
('beforehand'),
('begin'),
('beginning'),
('beginnings'),
('begins'),
('behind'),
('being'),
('believe'),
('below'),
('beside'),
('besides'),
('between'),
('beyond'),
('biol'),
('both'),
('brief'),
('briefly'),
('but'),
('by'),
('c'),
('ca'),
('came'),
('can'),
('cannot'),
('can''t'),
('cause'),
('causes'),
('certain'),
('certainly'),
('co'),
('com'),
('come'),
('comes'),
('contain'),
('containing'),
('contains'),
('could'),
('couldnt'),
('d'),
('date'),
('did'),
('didn''t'),
('different'),
('do'),
('does'),
('doesn''t'),
('doing'),
('done'),
('don''t'),
('down'),
('downwards'),
('due'),
('during'),
('e'),
('each'),
('ed'),
('edu'),
('effect'),
('eg'),
('eight'),
('eighty'),
('either'),
('else'),
('elsewhere'),
('end'),
('ending'),
('enough'),
('especially'),
('et'),
('et-al'),
('etc'),
('even'),
('ever'),
('every'),
('everybody'),
('everyone'),
('everything'),
('everywhere'),
('ex'),
('except'),
('f'),
('far'),
('few'),
('ff'),
('fifth'),
('first'),
('five'),
('fix'),
('followed'),
('following'),
('follows'),
('for'),
('former'),
('formerly'),
('forth'),
('found'),
('four'),
('from'),
('further'),
('furthermore'),
('g'),
('gave'),
('get'),
('gets'),
('getting'),
('give'),
('given'),
('gives'),
('giving'),
('go'),
('goes'),
('gone'),
('got'),
('gotten'),
('h'),
('had'),
('happens'),
('hardly'),
('has'),
('hasn''t'),
('have'),
('haven''t'),
('having'),
('he'),
('hed'),
('hence'),
('her'),
('here'),
('hereafter'),
('hereby'),
('herein'),
('heres'),
('hereupon'),
('hers'),
('herself'),
('hes'),
('hi'),
('hid'),
('him'),
('himself'),
('his'),
('hither'),
('home'),
('how'),
('howbeit'),
('however'),
('hundred'),
('i'),
('id'),
('ie'),
('if'),
('i''ll'),
('im'),
('immediate'),
('immediately'),
('importance'),
('important'),
('in'),
('inc'),
('indeed'),
('index'),
('information'),
('instead'),
('into'),
('invention'),
('inward'),
('is'),
('isn''t'),
('it'),
('itd'),
('it''ll'),
('its'),
('itself'),
('i''ve'),
('j'),
('just'),
('k'),
('keep keeps'),
('kept'),
('kg'),
('km'),
('know'),
('known'),
('knows'),
('l'),
('largely'),
('last'),
('lately'),
('later'),
('latter'),
('latterly'),
('least'),
('less'),
('lest'),
('let'),
('lets'),
('like'),
('liked'),
('likely'),
('line'),
('little'),
('''ll'),
('look'),
('looking'),
('looks'),
('ltd'),
('m'),
('made'),
('mainly'),
('make'),
('makes'),
('many'),
('may'),
('maybe'),
('me'),
('mean'),
('means'),
('meantime'),
('meanwhile'),
('merely'),
('mg'),
('might'),
('million'),
('miss'),
('ml'),
('more'),
('moreover'),
('most'),
('mostly'),
('mr'),
('mrs'),
('much'),
('mug'),
('must'),
('my'),
('myself'),
('n'),
('na'),
('name'),
('namely'),
('nay'),
('nd'),
('near'),
('nearly'),
('necessarily'),
('necessary'),
('need'),
('needs'),
('neither'),
('never'),
('nevertheless'),
('new'),
('next'),
('nine'),
('ninety'),
('no'),
('nobody'),
('non'),
('none'),
('nonetheless'),
('noone'),
('nor'),
('normally'),
('nos'),
('not'),
('noted'),
('nothing'),
('now'),
('nowhere'),
('o'),
('obtain'),
('obtained'),
('obviously'),
('of'),
('off'),
('often'),
('oh'),
('ok'),
('okay'),
('old'),
('omitted'),
('on'),
('once'),
('one'),
('ones'),
('only'),
('onto'),
('or'),
('ord'),
('other'),
('others'),
('otherwise'),
('ought'),
('our'),
('ours'),
('ourselves'),
('out'),
('outside'),
('over'),
('overall'),
('owing'),
('own'),
('p'),
('page'),
('pages'),
('part'),
('particular'),
('particularly'),
('past'),
('per'),
('perhaps'),
('placed'),
('please'),
('plus'),
('poorly'),
('possible'),
('possibly'),
('potentially'),
('pp'),
('predominantly'),
('present'),
('previously'),
('primarily'),
('probably'),
('promptly'),
('proud'),
('provides'),
('put'),
('q'),
('que'),
('quickly'),
('quite'),
('qv'),
('r'),
('ran'),
('rather'),
('rd'),
('re'),
('readily'),
('really'),
('recent'),
('recently'),
('ref'),
('refs'),
('regarding'),
('regardless'),
('regards'),
('related'),
('relatively'),
('research'),
('respectively'),
('resulted'),
('resulting'),
('results'),
('right'),
('run'),
('s'),
('said'),
('same'),
('saw'),
('say'),
('saying'),
('says'),
('sec'),
('section'),
('see'),
('seeing'),
('seem'),
('seemed'),
('seeming'),
('seems'),
('seen'),
('self'),
('selves'),
('sent'),
('seven'),
('several'),
('shall'),
('she'),
('shed'),
('she''ll'),
('shes'),
('should'),
('shouldn''t'),
('show'),
('showed'),
('shown'),
('showns'),
('shows'),
('significant'),
('significantly'),
('similar'),
('similarly'),
('since'),
('six'),
('slightly'),
('so'),
('some'),
('somebody'),
('somehow'),
('someone'),
('somethan'),
('something'),
('sometime'),
('sometimes'),
('somewhat'),
('somewhere'),
('soon'),
('sorry'),
('specifically'),
('specified'),
('specify'),
('specifying'),
('still'),
('stop'),
('strongly'),
('sub'),
('substantially'),
('successfully'),
('such'),
('sufficiently'),
('suggest'),
('sup'),
('sure t'),
('take'),
('taken'),
('taking'),
('tell'),
('tends'),
('th'),
('than'),
('thank'),
('thanks'),
('thanx'),
('that'),
('that''ll'),
('thats'),
('that''ve'),
('the'),
('their'),
('theirs'),
('them'),
('themselves'),
('then'),
('thence'),
('there'),
('thereafter'),
('thereby'),
('thered'),
('therefore'),
('therein'),
('there''ll'),
('thereof'),
('therere'),
('theres'),
('thereto'),
('thereupon'),
('there''ve'),
('these'),
('they'),
('theyd'),
('they''ll'),
('theyre'),
('they''ve'),
('think'),
('this'),
('those'),
('thou'),
('though'),
('thoughh'),
('thousand'),
('throug'),
('through'),
('throughout'),
('thru'),
('thus'),
('til'),
('tip'),
('to'),
('together'),
('too'),
('took'),
('toward'),
('towards'),
('tried'),
('tries'),
('truly'),
('try'),
('trying'),
('ts'),
('twice'),
('two'),
('u'),
('un'),
('under'),
('unfortunately'),
('unless'),
('unlike'),
('unlikely'),
('until'),
('unto'),
('up'),
('upon'),
('ups'),
('us'),
('use'),
('used'),
('useful'),
('usefully'),
('usefulness'),
('uses'),
('using'),
('usually'),
('v'),
('value'),
('various'),
('''ve'),
('very'),
('via'),
('viz'),
('vol'),
('vols'),
('vs'),
('w'),
('want'),
('wants'),
('was'),
('wasnt'),
('way'),
('we'),
('wed'),
('welcome'),
('we''ll'),
('went'),
('were'),
('werent'),
('we''ve'),
('what'),
('whatever'),
('what''ll'),
('whats'),
('when'),
('whence'),
('whenever'),
('where'),
('whereafter'),
('whereas'),
('whereby'),
('wherein'),
('wheres'),
('whereupon'),
('wherever'),
('whether'),
('which'),
('while'),
('whim'),
('whither'),
('who'),
('whod'),
('whoever'),
('whole'),
('who''ll'),
('whom'),
('whomever'),
('whos'),
('whose'),
('why'),
('widely'),
('willing'),
('wish'),
('with'),
('within'),
('without'),
('wont'),
('words'),
('world'),
('would'),
('wouldnt'),
('www'),
('x'),
('y'),
('yes'),
('yet'),
('you'),
('youd'),
('you''ll'),
('your'),
('youre'),
('yours'),
('yourself'),
('yourselves'),
('you''ve'),
('z'),
('zero');

12
fedilogue/Dockerfile Normal file
View File

@ -0,0 +1,12 @@
FROM golang:latest AS build
WORKDIR /build
COPY go.mod go.sum .
RUN go mod download
COPY . .
RUN go build .
FROM alpine:latest AS run
WORKDIR /app
RUN apk add gcompat
COPY --from=build /build/fedilogue /app/fedilogue
ENTRYPOINT ["/app/fedilogue"]

View File

@ -2,6 +2,8 @@ package main
import ( import (
"flag" "flag"
"fmt"
) )
// Settings - Configuration file structure // Settings - Configuration file structure
@ -27,5 +29,9 @@ func getSettings() {
flag.BoolVar(&settings.Crawl, "c", true, "Crawl mode (default is yes)") flag.BoolVar(&settings.Crawl, "c", true, "Crawl mode (default is yes)")
flag.StringVar(&settings.Hostname, "h", "myhostname", "Set your hostname") flag.StringVar(&settings.Hostname, "h", "myhostname", "Set your hostname")
flag.IntVar(&settings.LogLevel, "l", 1, "Logging Level:\n 0) No logs\n 1) Reports every 30 seconds\n 2) Errors\n 3) Warnings\n 4) New Connections\n 5) Debugging\n") flag.IntVar(&settings.LogLevel, "l", 1, "Logging Level:\n 0) No logs\n 1) Reports every 30 seconds\n 2) Errors\n 3) Warnings\n 4) New Connections\n 5) Debugging\n")
fmt.Println("Crawl: ", settings.Crawl)
flag.Parse() flag.Parse()
fmt.Println(settings)
} }

View File

@ -11,7 +11,7 @@ var pool *pgxpool.Pool
func getDbPool() *pgxpool.Pool { func getDbPool() *pgxpool.Pool {
// Setup Database // Setup Database
dburl := os.Getenv("DATABASE_URL") dburl := "postgres://" + os.Getenv("POSTGRES_USER") + ":" + os.Getenv("POSTGRES_PASSWORD") + "@" + os.Getenv("POSTGRES_HOSTNAME") + "/" + os.Getenv("POSTGRES_DB")
pool, err := pgxpool.Connect(context.Background(), dburl) pool, err := pgxpool.Connect(context.Background(), dburl)
if err != nil { if err != nil {
logFatal("Unable to connect to database:", err) logFatal("Unable to connect to database:", err)

View File

@ -1,4 +1,4 @@
module git.farhan.codes/farhan/fedilogue/fedilogger module git.farhan.codes/farhan/fedilogue/fedilogue
go 1.23.5 go 1.23.5

View File

@ -92,6 +92,7 @@ func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningIn
pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json" pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"
// Checking Mastodon and Pleroma (with .json) // Checking Mastodon and Pleroma (with .json)
logDebug("GetInstanceInfo: For " + endpoint + " Mastodon/Pleroma URL: " + pleromastodon_nodeinfo_uri)
reqjson, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil) reqjson, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil)
reqjson.Header.Set("User-Agent", "Tusky") reqjson.Header.Set("User-Agent", "Tusky")
@ -116,6 +117,7 @@ func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningIn
// Checking for Misskey (without .json) // Checking for Misskey (without .json)
misskey_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0" misskey_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0"
logDebug("GetInstanceInfo: For " + endpoint + " MissKey URL: " + misskey_nodeinfo_uri)
req, _ := http.NewRequest("GET", misskey_nodeinfo_uri, nil) req, _ := http.NewRequest("GET", misskey_nodeinfo_uri, nil)
req.Header.Set("User-Agent", "Tusky") req.Header.Set("User-Agent", "Tusky")
@ -140,6 +142,7 @@ func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningIn
// Check for Lemmy (With Json) // Check for Lemmy (With Json)
lemmy_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.1" lemmy_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.1"
logDebug("GetInstanceInfo: For " + endpoint + " Lemmy URL: " + lemmy_nodeinfo_uri)
req, _ = http.NewRequest("GET", lemmy_nodeinfo_uri, nil) req, _ = http.NewRequest("GET", lemmy_nodeinfo_uri, nil)
req.Header.Set("User-Agent", "Tusky") req.Header.Set("User-Agent", "Tusky")
@ -227,7 +230,10 @@ func CheckInstance(newinstance string, callerEndpoint string) {
go StartInstance(newinstance) go StartInstance(newinstance)
} }
ri_mutex.Unlock() ri_mutex.Unlock()
} else {
logDebug("Not crawling into ", newinstance)
} }
} }
func staggeredStart() { func staggeredStart() {
@ -257,7 +263,6 @@ func StartInstance(endpoint string) {
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version) logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
o.CaptureType = "Stream" o.CaptureType = "Stream"
UpdateRunner(endpoint, o) UpdateRunner(endpoint, o)
// PollMastodonPleroma(endpoint, &o)
StreamPleroma(endpoint) StreamPleroma(endpoint)
} else if o.Software == "mastodon" { } else if o.Software == "mastodon" {
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version) logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)

View File

@ -45,7 +45,7 @@ type MisskeyRequestParams struct {
} }
func StreamMisskey(endpoint string) { func StreamMisskey(endpoint string) {
logDebug("StreamMisskey for ", endpoint) logDebug("Stream Misskey for ", endpoint)
u := url.URL{ u := url.URL{
Scheme: "wss", Scheme: "wss",
Host: endpoint, Host: endpoint,
@ -200,6 +200,7 @@ func StreamPleroma(endpoint string) {
} }
func StreamMastodon(endpoint string, o *shared.RunningInstance) { func StreamMastodon(endpoint string, o *shared.RunningInstance) {
logDebug("Stream Mastodon for ", endpoint)
stream_client := BuildClient(endpoint) stream_client := BuildClient(endpoint)
var retry bool var retry bool
@ -207,6 +208,7 @@ func StreamMastodon(endpoint string, o *shared.RunningInstance) {
api_timeline := "https://" + endpoint + "/api/v1/streaming/public" api_timeline := "https://" + endpoint + "/api/v1/streaming/public"
for { for {
logDebug("StreamMastodon: " + endpoint + " URL: " + api_timeline)
req, err := http.NewRequest("GET", api_timeline, nil) req, err := http.NewRequest("GET", api_timeline, nil)
req.Header.Set("User-Agent", "Tusky") req.Header.Set("User-Agent", "Tusky")
if err != nil { if err != nil {

View File

@ -5,7 +5,8 @@ RUN go mod download
COPY . . COPY . .
RUN go build . RUN go build .
FROM alpine:latest FROM alpine:latest AS run
WORKDIR /app WORKDIR /app
RUN apk add gcompat
COPY --from=build /build/restapi /app/restapi COPY --from=build /build/restapi /app/restapi
ENTRYPOINT ["/app/restapi"] ENTRYPOINT ["/app/restapi"]

View File

@ -11,7 +11,8 @@ var pool *pgxpool.Pool
func getDbPool() *pgxpool.Pool { func getDbPool() *pgxpool.Pool {
// Setup Database // Setup Database
pool, err := pgxpool.Connect(context.Background(), os.Getenv("DATABASE_URL")) dburl := "postgres://" + os.Getenv("POSTGRES_USER") + ":" + os.Getenv("POSTGRES_PASSWORD") + "@" + os.Getenv("POSTGRES_HOSTNAME") + "/" + os.Getenv("POSTGRES_DB")
pool, err := pgxpool.Connect(context.Background(), dburl)
if err != nil { if err != nil {
log.Fatal("Unable to connect to database:", err) log.Fatal("Unable to connect to database:", err)
} }

View File

@ -1,5 +0,0 @@
#!/bin/sh
sleep 5
./restapi

View File

@ -383,7 +383,7 @@ func getGlobalGraph(w http.ResponseWriter, r *http.Request) {
line.SetXAxis(dates). line.SetXAxis(dates).
AddSeries("Post Count", generateLineItems(counts)). AddSeries("Post Count", generateLineItems(counts)).
SetSeriesOptions(charts.WithLineChartOpts(opts.LineChart{Smooth: false})) SetSeriesOptions(charts.WithLineChartOpts(opts.LineChart{Smooth: opts.Bool(false)}))
line.Render(w) line.Render(w)
} }