Compare commits
6 Commits
f3bbf9a1af
...
097c05b478
Author | SHA1 | Date | |
---|---|---|---|
097c05b478 | |||
808bac398a | |||
9a1b2c98a3 | |||
d3efb1c8dd | |||
c6ba6fdfde | |||
22646f052d |
@ -1,6 +1,11 @@
|
||||
services:
|
||||
fedilogue:
|
||||
build: fedilogger/
|
||||
build: fedilogue/
|
||||
environment:
|
||||
POSTGRES_USER: $POSTGRES_USER
|
||||
POSTGRES_PASSWORD: $POSTGRES_PASSWORD
|
||||
POSTGRES_HOSTNAME: $POSTGRES_HOSTNAME
|
||||
POSTGRES_DB: $POSTGRES_DB
|
||||
depends_on:
|
||||
- db
|
||||
ports:
|
||||
@ -10,16 +15,17 @@ services:
|
||||
ports:
|
||||
- "127.0.0.1:6431:6432"
|
||||
environment:
|
||||
POSTGRES_USER: $PGUSER
|
||||
POSTGRES_PASSWORD: $PGPASS
|
||||
POSTGRES_DB: $PGDATABASE
|
||||
POSTGRES_USER: $POSTGRES_USER
|
||||
POSTGRES_PASSWORD: $POSTGRES_PASSWORD
|
||||
POSTGRES_HOSTNAME: $POSTGRES_HOSTNAME
|
||||
POSTGRES_DB: $POSTGRES_DB
|
||||
depends_on:
|
||||
- db
|
||||
db:
|
||||
image: postgres:alpine
|
||||
environment:
|
||||
POSTGRES_USER: $PGUSER
|
||||
POSTGRES_PASSWORD: $PGPASS
|
||||
POSTGRES_DB: $PGDATABASE
|
||||
POSTGRES_USER: $POSTGRES_USER
|
||||
POSTGRES_PASSWORD: $POSTGRES_PASSWORD
|
||||
POSTGRES_DB: $POSTGRES_DB
|
||||
volumes:
|
||||
- ./postgres-data:/var/lib/postgresql
|
||||
|
@ -1,11 +0,0 @@
|
||||
FROM golang:latest AS build
|
||||
WORKDIR /build
|
||||
COPY go.mod go.sum .
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go build .
|
||||
|
||||
FROM alpine:latest
|
||||
WORKDIR /app
|
||||
COPY --from=build /build/fedilogger /app/fedilogger
|
||||
ENTRYPOINT ["/app/restapi"]
|
@ -1,731 +0,0 @@
|
||||
CREATE TABLE IF NOT EXISTS actors (
|
||||
id SERIAL PRIMARY KEY,
|
||||
document JSONB,
|
||||
identifiedat TIMESTAMP with time zone DEFAULT now(),
|
||||
instance VARCHAR(1000) NOT NULL,
|
||||
bot BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS activities (
|
||||
id SERIAL PRIMARY KEY,
|
||||
document JSONB,
|
||||
normalized TEXT,
|
||||
identifiedat TIMESTAMP with time zone DEFAULT now(),
|
||||
instance VARCHAR(1000) NOT NULL,
|
||||
hashtags VARCHAR(140)[],
|
||||
bot BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS instances (
|
||||
endpoint VARCHAR(2083) NOT NULL PRIMARY KEY UNIQUE,
|
||||
state VARCHAR(16),
|
||||
username VARCHAR(32),
|
||||
password VARCHAR(32),
|
||||
software VARCHAR(50),
|
||||
banned BOOLEAN DEFAULT FALSE,
|
||||
alwaysbot BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
|
||||
-- Autostart mastodon.social
|
||||
INSERT INTO instances (endpoint) VALUES('mastodon.social');
|
||||
-- Banned Instances
|
||||
INSERT INTO instances (endpoint, banned) VALUES
|
||||
('switter.at', true),
|
||||
('xxxtumblr.org', true),
|
||||
('sinblr.com', true),
|
||||
('twitiverse.com', true),
|
||||
('my.dirtyhobby.xyz', true),
|
||||
('bae.st', true);
|
||||
-- Alwaysbot instances
|
||||
INSERT INTO instances (endpoint, alwaysbot) VALUES
|
||||
('mstdn.foxfam.club', true),
|
||||
('botsin.space', true),
|
||||
('newsbots.eu', true);
|
||||
|
||||
ALTER TABLE activities
|
||||
ADD normalized_tsvector tsvector
|
||||
GENERATED ALWAYS AS (to_tsvector('english', normalized)) STORED;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS actors_uri_idx ON actors ( (document->>'id') );
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS activities_uri_idx ON activities ( (document->>'id') );
|
||||
|
||||
CREATE INDEX IF NOT EXISTS activities_published_idx ON activities ( (document->>'published') );
|
||||
CREATE INDEX IF NOT EXISTS activities_identifiedat_idx ON activities (identifiedat);
|
||||
CREATE INDEX IF NOT EXISTS hashtags_idx ON activities(hashtags);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS normalized_idx ON activities USING gin(normalized_tsvector);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS actors_id_idx ON actors (id);
|
||||
CREATE INDEX IF NOT EXISTS activities_id_idx ON activities (id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stopwords (
|
||||
word VARCHAR(20)
|
||||
);
|
||||
|
||||
INSERT INTO stopwords (word)
|
||||
VALUES
|
||||
('a'),
|
||||
('able'),
|
||||
('about'),
|
||||
('above'),
|
||||
('abst'),
|
||||
('accordance'),
|
||||
('according'),
|
||||
('accordingly'),
|
||||
('across'),
|
||||
('act'),
|
||||
('actually'),
|
||||
('added'),
|
||||
('adj'),
|
||||
('affected'),
|
||||
('affecting'),
|
||||
('affects'),
|
||||
('after'),
|
||||
('afterwards'),
|
||||
('again'),
|
||||
('against'),
|
||||
('ah'),
|
||||
('all'),
|
||||
('almost'),
|
||||
('alone'),
|
||||
('along'),
|
||||
('already'),
|
||||
('also'),
|
||||
('although'),
|
||||
('always'),
|
||||
('am'),
|
||||
('among'),
|
||||
('amongst'),
|
||||
('an'),
|
||||
('and'),
|
||||
('announce'),
|
||||
('another'),
|
||||
('any'),
|
||||
('anybody'),
|
||||
('anyhow'),
|
||||
('anymore'),
|
||||
('anyone'),
|
||||
('anything'),
|
||||
('anyway'),
|
||||
('anyways'),
|
||||
('anywhere'),
|
||||
('apparently'),
|
||||
('approximately'),
|
||||
('are'),
|
||||
('aren'),
|
||||
('arent'),
|
||||
('arise'),
|
||||
('around'),
|
||||
('as'),
|
||||
('aside'),
|
||||
('ask'),
|
||||
('asking'),
|
||||
('at'),
|
||||
('auth'),
|
||||
('available'),
|
||||
('away'),
|
||||
('awfully'),
|
||||
('b'),
|
||||
('back'),
|
||||
('be'),
|
||||
('became'),
|
||||
('because'),
|
||||
('become'),
|
||||
('becomes'),
|
||||
('becoming'),
|
||||
('been'),
|
||||
('before'),
|
||||
('beforehand'),
|
||||
('begin'),
|
||||
('beginning'),
|
||||
('beginnings'),
|
||||
('begins'),
|
||||
('behind'),
|
||||
('being'),
|
||||
('believe'),
|
||||
('below'),
|
||||
('beside'),
|
||||
('besides'),
|
||||
('between'),
|
||||
('beyond'),
|
||||
('biol'),
|
||||
('both'),
|
||||
('brief'),
|
||||
('briefly'),
|
||||
('but'),
|
||||
('by'),
|
||||
('c'),
|
||||
('ca'),
|
||||
('came'),
|
||||
('can'),
|
||||
('cannot'),
|
||||
('can''t'),
|
||||
('cause'),
|
||||
('causes'),
|
||||
('certain'),
|
||||
('certainly'),
|
||||
('co'),
|
||||
('com'),
|
||||
('come'),
|
||||
('comes'),
|
||||
('contain'),
|
||||
('containing'),
|
||||
('contains'),
|
||||
('could'),
|
||||
('couldnt'),
|
||||
('d'),
|
||||
('date'),
|
||||
('did'),
|
||||
('didn''t'),
|
||||
('different'),
|
||||
('do'),
|
||||
('does'),
|
||||
('doesn''t'),
|
||||
('doing'),
|
||||
('done'),
|
||||
('don''t'),
|
||||
('down'),
|
||||
('downwards'),
|
||||
('due'),
|
||||
('during'),
|
||||
('e'),
|
||||
('each'),
|
||||
('ed'),
|
||||
('edu'),
|
||||
('effect'),
|
||||
('eg'),
|
||||
('eight'),
|
||||
('eighty'),
|
||||
('either'),
|
||||
('else'),
|
||||
('elsewhere'),
|
||||
('end'),
|
||||
('ending'),
|
||||
('enough'),
|
||||
('especially'),
|
||||
('et'),
|
||||
('et-al'),
|
||||
('etc'),
|
||||
('even'),
|
||||
('ever'),
|
||||
('every'),
|
||||
('everybody'),
|
||||
('everyone'),
|
||||
('everything'),
|
||||
('everywhere'),
|
||||
('ex'),
|
||||
('except'),
|
||||
('f'),
|
||||
('far'),
|
||||
('few'),
|
||||
('ff'),
|
||||
('fifth'),
|
||||
('first'),
|
||||
('five'),
|
||||
('fix'),
|
||||
('followed'),
|
||||
('following'),
|
||||
('follows'),
|
||||
('for'),
|
||||
('former'),
|
||||
('formerly'),
|
||||
('forth'),
|
||||
('found'),
|
||||
('four'),
|
||||
('from'),
|
||||
('further'),
|
||||
('furthermore'),
|
||||
('g'),
|
||||
('gave'),
|
||||
('get'),
|
||||
('gets'),
|
||||
('getting'),
|
||||
('give'),
|
||||
('given'),
|
||||
('gives'),
|
||||
('giving'),
|
||||
('go'),
|
||||
('goes'),
|
||||
('gone'),
|
||||
('got'),
|
||||
('gotten'),
|
||||
('h'),
|
||||
('had'),
|
||||
('happens'),
|
||||
('hardly'),
|
||||
('has'),
|
||||
('hasn''t'),
|
||||
('have'),
|
||||
('haven''t'),
|
||||
('having'),
|
||||
('he'),
|
||||
('hed'),
|
||||
('hence'),
|
||||
('her'),
|
||||
('here'),
|
||||
('hereafter'),
|
||||
('hereby'),
|
||||
('herein'),
|
||||
('heres'),
|
||||
('hereupon'),
|
||||
('hers'),
|
||||
('herself'),
|
||||
('hes'),
|
||||
('hi'),
|
||||
('hid'),
|
||||
('him'),
|
||||
('himself'),
|
||||
('his'),
|
||||
('hither'),
|
||||
('home'),
|
||||
('how'),
|
||||
('howbeit'),
|
||||
('however'),
|
||||
('hundred'),
|
||||
('i'),
|
||||
('id'),
|
||||
('ie'),
|
||||
('if'),
|
||||
('i''ll'),
|
||||
('im'),
|
||||
('immediate'),
|
||||
('immediately'),
|
||||
('importance'),
|
||||
('important'),
|
||||
('in'),
|
||||
('inc'),
|
||||
('indeed'),
|
||||
('index'),
|
||||
('information'),
|
||||
('instead'),
|
||||
('into'),
|
||||
('invention'),
|
||||
('inward'),
|
||||
('is'),
|
||||
('isn''t'),
|
||||
('it'),
|
||||
('itd'),
|
||||
('it''ll'),
|
||||
('its'),
|
||||
('itself'),
|
||||
('i''ve'),
|
||||
('j'),
|
||||
('just'),
|
||||
('k'),
|
||||
('keep keeps'),
|
||||
('kept'),
|
||||
('kg'),
|
||||
('km'),
|
||||
('know'),
|
||||
('known'),
|
||||
('knows'),
|
||||
('l'),
|
||||
('largely'),
|
||||
('last'),
|
||||
('lately'),
|
||||
('later'),
|
||||
('latter'),
|
||||
('latterly'),
|
||||
('least'),
|
||||
('less'),
|
||||
('lest'),
|
||||
('let'),
|
||||
('lets'),
|
||||
('like'),
|
||||
('liked'),
|
||||
('likely'),
|
||||
('line'),
|
||||
('little'),
|
||||
('''ll'),
|
||||
('look'),
|
||||
('looking'),
|
||||
('looks'),
|
||||
('ltd'),
|
||||
('m'),
|
||||
('made'),
|
||||
('mainly'),
|
||||
('make'),
|
||||
('makes'),
|
||||
('many'),
|
||||
('may'),
|
||||
('maybe'),
|
||||
('me'),
|
||||
('mean'),
|
||||
('means'),
|
||||
('meantime'),
|
||||
('meanwhile'),
|
||||
('merely'),
|
||||
('mg'),
|
||||
('might'),
|
||||
('million'),
|
||||
('miss'),
|
||||
('ml'),
|
||||
('more'),
|
||||
('moreover'),
|
||||
('most'),
|
||||
('mostly'),
|
||||
('mr'),
|
||||
('mrs'),
|
||||
('much'),
|
||||
('mug'),
|
||||
('must'),
|
||||
('my'),
|
||||
('myself'),
|
||||
('n'),
|
||||
('na'),
|
||||
('name'),
|
||||
('namely'),
|
||||
('nay'),
|
||||
('nd'),
|
||||
('near'),
|
||||
('nearly'),
|
||||
('necessarily'),
|
||||
('necessary'),
|
||||
('need'),
|
||||
('needs'),
|
||||
('neither'),
|
||||
('never'),
|
||||
('nevertheless'),
|
||||
('new'),
|
||||
('next'),
|
||||
('nine'),
|
||||
('ninety'),
|
||||
('no'),
|
||||
('nobody'),
|
||||
('non'),
|
||||
('none'),
|
||||
('nonetheless'),
|
||||
('noone'),
|
||||
('nor'),
|
||||
('normally'),
|
||||
('nos'),
|
||||
('not'),
|
||||
('noted'),
|
||||
('nothing'),
|
||||
('now'),
|
||||
('nowhere'),
|
||||
('o'),
|
||||
('obtain'),
|
||||
('obtained'),
|
||||
('obviously'),
|
||||
('of'),
|
||||
('off'),
|
||||
('often'),
|
||||
('oh'),
|
||||
('ok'),
|
||||
('okay'),
|
||||
('old'),
|
||||
('omitted'),
|
||||
('on'),
|
||||
('once'),
|
||||
('one'),
|
||||
('ones'),
|
||||
('only'),
|
||||
('onto'),
|
||||
('or'),
|
||||
('ord'),
|
||||
('other'),
|
||||
('others'),
|
||||
('otherwise'),
|
||||
('ought'),
|
||||
('our'),
|
||||
('ours'),
|
||||
('ourselves'),
|
||||
('out'),
|
||||
('outside'),
|
||||
('over'),
|
||||
('overall'),
|
||||
('owing'),
|
||||
('own'),
|
||||
('p'),
|
||||
('page'),
|
||||
('pages'),
|
||||
('part'),
|
||||
('particular'),
|
||||
('particularly'),
|
||||
('past'),
|
||||
('per'),
|
||||
('perhaps'),
|
||||
('placed'),
|
||||
('please'),
|
||||
('plus'),
|
||||
('poorly'),
|
||||
('possible'),
|
||||
('possibly'),
|
||||
('potentially'),
|
||||
('pp'),
|
||||
('predominantly'),
|
||||
('present'),
|
||||
('previously'),
|
||||
('primarily'),
|
||||
('probably'),
|
||||
('promptly'),
|
||||
('proud'),
|
||||
('provides'),
|
||||
('put'),
|
||||
('q'),
|
||||
('que'),
|
||||
('quickly'),
|
||||
('quite'),
|
||||
('qv'),
|
||||
('r'),
|
||||
('ran'),
|
||||
('rather'),
|
||||
('rd'),
|
||||
('re'),
|
||||
('readily'),
|
||||
('really'),
|
||||
('recent'),
|
||||
('recently'),
|
||||
('ref'),
|
||||
('refs'),
|
||||
('regarding'),
|
||||
('regardless'),
|
||||
('regards'),
|
||||
('related'),
|
||||
('relatively'),
|
||||
('research'),
|
||||
('respectively'),
|
||||
('resulted'),
|
||||
('resulting'),
|
||||
('results'),
|
||||
('right'),
|
||||
('run'),
|
||||
('s'),
|
||||
('said'),
|
||||
('same'),
|
||||
('saw'),
|
||||
('say'),
|
||||
('saying'),
|
||||
('says'),
|
||||
('sec'),
|
||||
('section'),
|
||||
('see'),
|
||||
('seeing'),
|
||||
('seem'),
|
||||
('seemed'),
|
||||
('seeming'),
|
||||
('seems'),
|
||||
('seen'),
|
||||
('self'),
|
||||
('selves'),
|
||||
('sent'),
|
||||
('seven'),
|
||||
('several'),
|
||||
('shall'),
|
||||
('she'),
|
||||
('shed'),
|
||||
('she''ll'),
|
||||
('shes'),
|
||||
('should'),
|
||||
('shouldn''t'),
|
||||
('show'),
|
||||
('showed'),
|
||||
('shown'),
|
||||
('showns'),
|
||||
('shows'),
|
||||
('significant'),
|
||||
('significantly'),
|
||||
('similar'),
|
||||
('similarly'),
|
||||
('since'),
|
||||
('six'),
|
||||
('slightly'),
|
||||
('so'),
|
||||
('some'),
|
||||
('somebody'),
|
||||
('somehow'),
|
||||
('someone'),
|
||||
('somethan'),
|
||||
('something'),
|
||||
('sometime'),
|
||||
('sometimes'),
|
||||
('somewhat'),
|
||||
('somewhere'),
|
||||
('soon'),
|
||||
('sorry'),
|
||||
('specifically'),
|
||||
('specified'),
|
||||
('specify'),
|
||||
('specifying'),
|
||||
('still'),
|
||||
('stop'),
|
||||
('strongly'),
|
||||
('sub'),
|
||||
('substantially'),
|
||||
('successfully'),
|
||||
('such'),
|
||||
('sufficiently'),
|
||||
('suggest'),
|
||||
('sup'),
|
||||
('sure t'),
|
||||
('take'),
|
||||
('taken'),
|
||||
('taking'),
|
||||
('tell'),
|
||||
('tends'),
|
||||
('th'),
|
||||
('than'),
|
||||
('thank'),
|
||||
('thanks'),
|
||||
('thanx'),
|
||||
('that'),
|
||||
('that''ll'),
|
||||
('thats'),
|
||||
('that''ve'),
|
||||
('the'),
|
||||
('their'),
|
||||
('theirs'),
|
||||
('them'),
|
||||
('themselves'),
|
||||
('then'),
|
||||
('thence'),
|
||||
('there'),
|
||||
('thereafter'),
|
||||
('thereby'),
|
||||
('thered'),
|
||||
('therefore'),
|
||||
('therein'),
|
||||
('there''ll'),
|
||||
('thereof'),
|
||||
('therere'),
|
||||
('theres'),
|
||||
('thereto'),
|
||||
('thereupon'),
|
||||
('there''ve'),
|
||||
('these'),
|
||||
('they'),
|
||||
('theyd'),
|
||||
('they''ll'),
|
||||
('theyre'),
|
||||
('they''ve'),
|
||||
('think'),
|
||||
('this'),
|
||||
('those'),
|
||||
('thou'),
|
||||
('though'),
|
||||
('thoughh'),
|
||||
('thousand'),
|
||||
('throug'),
|
||||
('through'),
|
||||
('throughout'),
|
||||
('thru'),
|
||||
('thus'),
|
||||
('til'),
|
||||
('tip'),
|
||||
('to'),
|
||||
('together'),
|
||||
('too'),
|
||||
('took'),
|
||||
('toward'),
|
||||
('towards'),
|
||||
('tried'),
|
||||
('tries'),
|
||||
('truly'),
|
||||
('try'),
|
||||
('trying'),
|
||||
('ts'),
|
||||
('twice'),
|
||||
('two'),
|
||||
('u'),
|
||||
('un'),
|
||||
('under'),
|
||||
('unfortunately'),
|
||||
('unless'),
|
||||
('unlike'),
|
||||
('unlikely'),
|
||||
('until'),
|
||||
('unto'),
|
||||
('up'),
|
||||
('upon'),
|
||||
('ups'),
|
||||
('us'),
|
||||
('use'),
|
||||
('used'),
|
||||
('useful'),
|
||||
('usefully'),
|
||||
('usefulness'),
|
||||
('uses'),
|
||||
('using'),
|
||||
('usually'),
|
||||
('v'),
|
||||
('value'),
|
||||
('various'),
|
||||
('''ve'),
|
||||
('very'),
|
||||
('via'),
|
||||
('viz'),
|
||||
('vol'),
|
||||
('vols'),
|
||||
('vs'),
|
||||
('w'),
|
||||
('want'),
|
||||
('wants'),
|
||||
('was'),
|
||||
('wasnt'),
|
||||
('way'),
|
||||
('we'),
|
||||
('wed'),
|
||||
('welcome'),
|
||||
('we''ll'),
|
||||
('went'),
|
||||
('were'),
|
||||
('werent'),
|
||||
('we''ve'),
|
||||
('what'),
|
||||
('whatever'),
|
||||
('what''ll'),
|
||||
('whats'),
|
||||
('when'),
|
||||
('whence'),
|
||||
('whenever'),
|
||||
('where'),
|
||||
('whereafter'),
|
||||
('whereas'),
|
||||
('whereby'),
|
||||
('wherein'),
|
||||
('wheres'),
|
||||
('whereupon'),
|
||||
('wherever'),
|
||||
('whether'),
|
||||
('which'),
|
||||
('while'),
|
||||
('whim'),
|
||||
('whither'),
|
||||
('who'),
|
||||
('whod'),
|
||||
('whoever'),
|
||||
('whole'),
|
||||
('who''ll'),
|
||||
('whom'),
|
||||
('whomever'),
|
||||
('whos'),
|
||||
('whose'),
|
||||
('why'),
|
||||
('widely'),
|
||||
('willing'),
|
||||
('wish'),
|
||||
('with'),
|
||||
('within'),
|
||||
('without'),
|
||||
('wont'),
|
||||
('words'),
|
||||
('world'),
|
||||
('would'),
|
||||
('wouldnt'),
|
||||
('www'),
|
||||
('x'),
|
||||
('y'),
|
||||
('yes'),
|
||||
('yet'),
|
||||
('you'),
|
||||
('youd'),
|
||||
('you''ll'),
|
||||
('your'),
|
||||
('youre'),
|
||||
('yours'),
|
||||
('yourself'),
|
||||
('yourselves'),
|
||||
('you''ve'),
|
||||
('z'),
|
||||
('zero');
|
12
fedilogue/Dockerfile
Normal file
12
fedilogue/Dockerfile
Normal file
@ -0,0 +1,12 @@
|
||||
FROM golang:latest AS build
|
||||
WORKDIR /build
|
||||
COPY go.mod go.sum .
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go build .
|
||||
|
||||
FROM alpine:latest AS run
|
||||
WORKDIR /app
|
||||
RUN apk add gcompat
|
||||
COPY --from=build /build/fedilogue /app/fedilogue
|
||||
ENTRYPOINT ["/app/fedilogue"]
|
@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
|
||||
)
|
||||
|
||||
// Settings - Configuration file structure
|
||||
@ -27,5 +29,9 @@ func getSettings() {
|
||||
flag.BoolVar(&settings.Crawl, "c", true, "Crawl mode (default is yes)")
|
||||
flag.StringVar(&settings.Hostname, "h", "myhostname", "Set your hostname")
|
||||
flag.IntVar(&settings.LogLevel, "l", 1, "Logging Level:\n 0) No logs\n 1) Reports every 30 seconds\n 2) Errors\n 3) Warnings\n 4) New Connections\n 5) Debugging\n")
|
||||
|
||||
fmt.Println("Crawl: ", settings.Crawl)
|
||||
|
||||
flag.Parse()
|
||||
fmt.Println(settings)
|
||||
}
|
@ -11,7 +11,7 @@ var pool *pgxpool.Pool
|
||||
|
||||
func getDbPool() *pgxpool.Pool {
|
||||
// Setup Database
|
||||
dburl := os.Getenv("DATABASE_URL")
|
||||
dburl := "postgres://" + os.Getenv("POSTGRES_USER") + ":" + os.Getenv("POSTGRES_PASSWORD") + "@" + os.Getenv("POSTGRES_HOSTNAME") + "/" + os.Getenv("POSTGRES_DB")
|
||||
pool, err := pgxpool.Connect(context.Background(), dburl)
|
||||
if err != nil {
|
||||
logFatal("Unable to connect to database:", err)
|
@ -1,4 +1,4 @@
|
||||
module git.farhan.codes/farhan/fedilogue/fedilogger
|
||||
module git.farhan.codes/farhan/fedilogue/fedilogue
|
||||
|
||||
go 1.23.5
|
||||
|
@ -92,6 +92,7 @@ func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningIn
|
||||
pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"
|
||||
|
||||
// Checking Mastodon and Pleroma (with .json)
|
||||
logDebug("GetInstanceInfo: For " + endpoint + " Mastodon/Pleroma URL: " + pleromastodon_nodeinfo_uri)
|
||||
reqjson, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil)
|
||||
reqjson.Header.Set("User-Agent", "Tusky")
|
||||
|
||||
@ -116,6 +117,7 @@ func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningIn
|
||||
|
||||
// Checking for Misskey (without .json)
|
||||
misskey_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0"
|
||||
logDebug("GetInstanceInfo: For " + endpoint + " MissKey URL: " + misskey_nodeinfo_uri)
|
||||
req, _ := http.NewRequest("GET", misskey_nodeinfo_uri, nil)
|
||||
req.Header.Set("User-Agent", "Tusky")
|
||||
|
||||
@ -140,6 +142,7 @@ func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningIn
|
||||
|
||||
// Check for Lemmy (With Json)
|
||||
lemmy_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.1"
|
||||
logDebug("GetInstanceInfo: For " + endpoint + " Lemmy URL: " + lemmy_nodeinfo_uri)
|
||||
req, _ = http.NewRequest("GET", lemmy_nodeinfo_uri, nil)
|
||||
req.Header.Set("User-Agent", "Tusky")
|
||||
|
||||
@ -227,7 +230,10 @@ func CheckInstance(newinstance string, callerEndpoint string) {
|
||||
go StartInstance(newinstance)
|
||||
}
|
||||
ri_mutex.Unlock()
|
||||
} else {
|
||||
logDebug("Not crawling into ", newinstance)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func staggeredStart() {
|
||||
@ -257,7 +263,6 @@ func StartInstance(endpoint string) {
|
||||
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
|
||||
o.CaptureType = "Stream"
|
||||
UpdateRunner(endpoint, o)
|
||||
// PollMastodonPleroma(endpoint, &o)
|
||||
StreamPleroma(endpoint)
|
||||
} else if o.Software == "mastodon" {
|
||||
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
|
@ -45,7 +45,7 @@ type MisskeyRequestParams struct {
|
||||
}
|
||||
|
||||
func StreamMisskey(endpoint string) {
|
||||
logDebug("StreamMisskey for ", endpoint)
|
||||
logDebug("Stream Misskey for ", endpoint)
|
||||
u := url.URL{
|
||||
Scheme: "wss",
|
||||
Host: endpoint,
|
||||
@ -200,6 +200,7 @@ func StreamPleroma(endpoint string) {
|
||||
}
|
||||
|
||||
func StreamMastodon(endpoint string, o *shared.RunningInstance) {
|
||||
logDebug("Stream Mastodon for ", endpoint)
|
||||
stream_client := BuildClient(endpoint)
|
||||
|
||||
var retry bool
|
||||
@ -207,6 +208,7 @@ func StreamMastodon(endpoint string, o *shared.RunningInstance) {
|
||||
api_timeline := "https://" + endpoint + "/api/v1/streaming/public"
|
||||
|
||||
for {
|
||||
logDebug("StreamMastodon: " + endpoint + " URL: " + api_timeline)
|
||||
req, err := http.NewRequest("GET", api_timeline, nil)
|
||||
req.Header.Set("User-Agent", "Tusky")
|
||||
if err != nil {
|
@ -5,7 +5,8 @@ RUN go mod download
|
||||
COPY . .
|
||||
RUN go build .
|
||||
|
||||
FROM alpine:latest
|
||||
FROM alpine:latest AS run
|
||||
WORKDIR /app
|
||||
RUN apk add gcompat
|
||||
COPY --from=build /build/restapi /app/restapi
|
||||
ENTRYPOINT ["/app/restapi"]
|
||||
|
@ -11,7 +11,8 @@ var pool *pgxpool.Pool
|
||||
|
||||
func getDbPool() *pgxpool.Pool {
|
||||
// Setup Database
|
||||
pool, err := pgxpool.Connect(context.Background(), os.Getenv("DATABASE_URL"))
|
||||
dburl := "postgres://" + os.Getenv("POSTGRES_USER") + ":" + os.Getenv("POSTGRES_PASSWORD") + "@" + os.Getenv("POSTGRES_HOSTNAME") + "/" + os.Getenv("POSTGRES_DB")
|
||||
pool, err := pgxpool.Connect(context.Background(), dburl)
|
||||
if err != nil {
|
||||
log.Fatal("Unable to connect to database:", err)
|
||||
}
|
||||
|
@ -1,5 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
sleep 5
|
||||
|
||||
./restapi
|
@ -383,7 +383,7 @@ func getGlobalGraph(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
line.SetXAxis(dates).
|
||||
AddSeries("Post Count", generateLineItems(counts)).
|
||||
SetSeriesOptions(charts.WithLineChartOpts(opts.LineChart{Smooth: false}))
|
||||
SetSeriesOptions(charts.WithLineChartOpts(opts.LineChart{Smooth: opts.Bool(false)}))
|
||||
|
||||
line.Render(w)
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user