Compare commits
6 Commits
f3bbf9a1af
...
097c05b478
Author | SHA1 | Date | |
---|---|---|---|
097c05b478 | |||
808bac398a | |||
9a1b2c98a3 | |||
d3efb1c8dd | |||
c6ba6fdfde | |||
22646f052d |
@ -1,6 +1,11 @@
|
|||||||
services:
|
services:
|
||||||
fedilogue:
|
fedilogue:
|
||||||
build: fedilogger/
|
build: fedilogue/
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: $POSTGRES_USER
|
||||||
|
POSTGRES_PASSWORD: $POSTGRES_PASSWORD
|
||||||
|
POSTGRES_HOSTNAME: $POSTGRES_HOSTNAME
|
||||||
|
POSTGRES_DB: $POSTGRES_DB
|
||||||
depends_on:
|
depends_on:
|
||||||
- db
|
- db
|
||||||
ports:
|
ports:
|
||||||
@ -10,16 +15,17 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "127.0.0.1:6431:6432"
|
- "127.0.0.1:6431:6432"
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_USER: $PGUSER
|
POSTGRES_USER: $POSTGRES_USER
|
||||||
POSTGRES_PASSWORD: $PGPASS
|
POSTGRES_PASSWORD: $POSTGRES_PASSWORD
|
||||||
POSTGRES_DB: $PGDATABASE
|
POSTGRES_HOSTNAME: $POSTGRES_HOSTNAME
|
||||||
|
POSTGRES_DB: $POSTGRES_DB
|
||||||
depends_on:
|
depends_on:
|
||||||
- db
|
- db
|
||||||
db:
|
db:
|
||||||
image: postgres:alpine
|
image: postgres:alpine
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_USER: $PGUSER
|
POSTGRES_USER: $POSTGRES_USER
|
||||||
POSTGRES_PASSWORD: $PGPASS
|
POSTGRES_PASSWORD: $POSTGRES_PASSWORD
|
||||||
POSTGRES_DB: $PGDATABASE
|
POSTGRES_DB: $POSTGRES_DB
|
||||||
volumes:
|
volumes:
|
||||||
- ./postgres-data:/var/lib/postgresql
|
- ./postgres-data:/var/lib/postgresql
|
||||||
|
@ -1,11 +0,0 @@
|
|||||||
FROM golang:latest AS build
|
|
||||||
WORKDIR /build
|
|
||||||
COPY go.mod go.sum .
|
|
||||||
RUN go mod download
|
|
||||||
COPY . .
|
|
||||||
RUN go build .
|
|
||||||
|
|
||||||
FROM alpine:latest
|
|
||||||
WORKDIR /app
|
|
||||||
COPY --from=build /build/fedilogger /app/fedilogger
|
|
||||||
ENTRYPOINT ["/app/restapi"]
|
|
@ -1,731 +0,0 @@
|
|||||||
CREATE TABLE IF NOT EXISTS actors (
|
|
||||||
id SERIAL PRIMARY KEY,
|
|
||||||
document JSONB,
|
|
||||||
identifiedat TIMESTAMP with time zone DEFAULT now(),
|
|
||||||
instance VARCHAR(1000) NOT NULL,
|
|
||||||
bot BOOLEAN DEFAULT FALSE
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS activities (
|
|
||||||
id SERIAL PRIMARY KEY,
|
|
||||||
document JSONB,
|
|
||||||
normalized TEXT,
|
|
||||||
identifiedat TIMESTAMP with time zone DEFAULT now(),
|
|
||||||
instance VARCHAR(1000) NOT NULL,
|
|
||||||
hashtags VARCHAR(140)[],
|
|
||||||
bot BOOLEAN DEFAULT FALSE
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS instances (
|
|
||||||
endpoint VARCHAR(2083) NOT NULL PRIMARY KEY UNIQUE,
|
|
||||||
state VARCHAR(16),
|
|
||||||
username VARCHAR(32),
|
|
||||||
password VARCHAR(32),
|
|
||||||
software VARCHAR(50),
|
|
||||||
banned BOOLEAN DEFAULT FALSE,
|
|
||||||
alwaysbot BOOLEAN DEFAULT FALSE
|
|
||||||
);
|
|
||||||
|
|
||||||
-- Autostart mastodon.social
|
|
||||||
INSERT INTO instances (endpoint) VALUES('mastodon.social');
|
|
||||||
-- Banned Instances
|
|
||||||
INSERT INTO instances (endpoint, banned) VALUES
|
|
||||||
('switter.at', true),
|
|
||||||
('xxxtumblr.org', true),
|
|
||||||
('sinblr.com', true),
|
|
||||||
('twitiverse.com', true),
|
|
||||||
('my.dirtyhobby.xyz', true),
|
|
||||||
('bae.st', true);
|
|
||||||
-- Alwaysbot instances
|
|
||||||
INSERT INTO instances (endpoint, alwaysbot) VALUES
|
|
||||||
('mstdn.foxfam.club', true),
|
|
||||||
('botsin.space', true),
|
|
||||||
('newsbots.eu', true);
|
|
||||||
|
|
||||||
ALTER TABLE activities
|
|
||||||
ADD normalized_tsvector tsvector
|
|
||||||
GENERATED ALWAYS AS (to_tsvector('english', normalized)) STORED;
|
|
||||||
|
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS actors_uri_idx ON actors ( (document->>'id') );
|
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS activities_uri_idx ON activities ( (document->>'id') );
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS activities_published_idx ON activities ( (document->>'published') );
|
|
||||||
CREATE INDEX IF NOT EXISTS activities_identifiedat_idx ON activities (identifiedat);
|
|
||||||
CREATE INDEX IF NOT EXISTS hashtags_idx ON activities(hashtags);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS normalized_idx ON activities USING gin(normalized_tsvector);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS actors_id_idx ON actors (id);
|
|
||||||
CREATE INDEX IF NOT EXISTS activities_id_idx ON activities (id);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS stopwords (
|
|
||||||
word VARCHAR(20)
|
|
||||||
);
|
|
||||||
|
|
||||||
INSERT INTO stopwords (word)
|
|
||||||
VALUES
|
|
||||||
('a'),
|
|
||||||
('able'),
|
|
||||||
('about'),
|
|
||||||
('above'),
|
|
||||||
('abst'),
|
|
||||||
('accordance'),
|
|
||||||
('according'),
|
|
||||||
('accordingly'),
|
|
||||||
('across'),
|
|
||||||
('act'),
|
|
||||||
('actually'),
|
|
||||||
('added'),
|
|
||||||
('adj'),
|
|
||||||
('affected'),
|
|
||||||
('affecting'),
|
|
||||||
('affects'),
|
|
||||||
('after'),
|
|
||||||
('afterwards'),
|
|
||||||
('again'),
|
|
||||||
('against'),
|
|
||||||
('ah'),
|
|
||||||
('all'),
|
|
||||||
('almost'),
|
|
||||||
('alone'),
|
|
||||||
('along'),
|
|
||||||
('already'),
|
|
||||||
('also'),
|
|
||||||
('although'),
|
|
||||||
('always'),
|
|
||||||
('am'),
|
|
||||||
('among'),
|
|
||||||
('amongst'),
|
|
||||||
('an'),
|
|
||||||
('and'),
|
|
||||||
('announce'),
|
|
||||||
('another'),
|
|
||||||
('any'),
|
|
||||||
('anybody'),
|
|
||||||
('anyhow'),
|
|
||||||
('anymore'),
|
|
||||||
('anyone'),
|
|
||||||
('anything'),
|
|
||||||
('anyway'),
|
|
||||||
('anyways'),
|
|
||||||
('anywhere'),
|
|
||||||
('apparently'),
|
|
||||||
('approximately'),
|
|
||||||
('are'),
|
|
||||||
('aren'),
|
|
||||||
('arent'),
|
|
||||||
('arise'),
|
|
||||||
('around'),
|
|
||||||
('as'),
|
|
||||||
('aside'),
|
|
||||||
('ask'),
|
|
||||||
('asking'),
|
|
||||||
('at'),
|
|
||||||
('auth'),
|
|
||||||
('available'),
|
|
||||||
('away'),
|
|
||||||
('awfully'),
|
|
||||||
('b'),
|
|
||||||
('back'),
|
|
||||||
('be'),
|
|
||||||
('became'),
|
|
||||||
('because'),
|
|
||||||
('become'),
|
|
||||||
('becomes'),
|
|
||||||
('becoming'),
|
|
||||||
('been'),
|
|
||||||
('before'),
|
|
||||||
('beforehand'),
|
|
||||||
('begin'),
|
|
||||||
('beginning'),
|
|
||||||
('beginnings'),
|
|
||||||
('begins'),
|
|
||||||
('behind'),
|
|
||||||
('being'),
|
|
||||||
('believe'),
|
|
||||||
('below'),
|
|
||||||
('beside'),
|
|
||||||
('besides'),
|
|
||||||
('between'),
|
|
||||||
('beyond'),
|
|
||||||
('biol'),
|
|
||||||
('both'),
|
|
||||||
('brief'),
|
|
||||||
('briefly'),
|
|
||||||
('but'),
|
|
||||||
('by'),
|
|
||||||
('c'),
|
|
||||||
('ca'),
|
|
||||||
('came'),
|
|
||||||
('can'),
|
|
||||||
('cannot'),
|
|
||||||
('can''t'),
|
|
||||||
('cause'),
|
|
||||||
('causes'),
|
|
||||||
('certain'),
|
|
||||||
('certainly'),
|
|
||||||
('co'),
|
|
||||||
('com'),
|
|
||||||
('come'),
|
|
||||||
('comes'),
|
|
||||||
('contain'),
|
|
||||||
('containing'),
|
|
||||||
('contains'),
|
|
||||||
('could'),
|
|
||||||
('couldnt'),
|
|
||||||
('d'),
|
|
||||||
('date'),
|
|
||||||
('did'),
|
|
||||||
('didn''t'),
|
|
||||||
('different'),
|
|
||||||
('do'),
|
|
||||||
('does'),
|
|
||||||
('doesn''t'),
|
|
||||||
('doing'),
|
|
||||||
('done'),
|
|
||||||
('don''t'),
|
|
||||||
('down'),
|
|
||||||
('downwards'),
|
|
||||||
('due'),
|
|
||||||
('during'),
|
|
||||||
('e'),
|
|
||||||
('each'),
|
|
||||||
('ed'),
|
|
||||||
('edu'),
|
|
||||||
('effect'),
|
|
||||||
('eg'),
|
|
||||||
('eight'),
|
|
||||||
('eighty'),
|
|
||||||
('either'),
|
|
||||||
('else'),
|
|
||||||
('elsewhere'),
|
|
||||||
('end'),
|
|
||||||
('ending'),
|
|
||||||
('enough'),
|
|
||||||
('especially'),
|
|
||||||
('et'),
|
|
||||||
('et-al'),
|
|
||||||
('etc'),
|
|
||||||
('even'),
|
|
||||||
('ever'),
|
|
||||||
('every'),
|
|
||||||
('everybody'),
|
|
||||||
('everyone'),
|
|
||||||
('everything'),
|
|
||||||
('everywhere'),
|
|
||||||
('ex'),
|
|
||||||
('except'),
|
|
||||||
('f'),
|
|
||||||
('far'),
|
|
||||||
('few'),
|
|
||||||
('ff'),
|
|
||||||
('fifth'),
|
|
||||||
('first'),
|
|
||||||
('five'),
|
|
||||||
('fix'),
|
|
||||||
('followed'),
|
|
||||||
('following'),
|
|
||||||
('follows'),
|
|
||||||
('for'),
|
|
||||||
('former'),
|
|
||||||
('formerly'),
|
|
||||||
('forth'),
|
|
||||||
('found'),
|
|
||||||
('four'),
|
|
||||||
('from'),
|
|
||||||
('further'),
|
|
||||||
('furthermore'),
|
|
||||||
('g'),
|
|
||||||
('gave'),
|
|
||||||
('get'),
|
|
||||||
('gets'),
|
|
||||||
('getting'),
|
|
||||||
('give'),
|
|
||||||
('given'),
|
|
||||||
('gives'),
|
|
||||||
('giving'),
|
|
||||||
('go'),
|
|
||||||
('goes'),
|
|
||||||
('gone'),
|
|
||||||
('got'),
|
|
||||||
('gotten'),
|
|
||||||
('h'),
|
|
||||||
('had'),
|
|
||||||
('happens'),
|
|
||||||
('hardly'),
|
|
||||||
('has'),
|
|
||||||
('hasn''t'),
|
|
||||||
('have'),
|
|
||||||
('haven''t'),
|
|
||||||
('having'),
|
|
||||||
('he'),
|
|
||||||
('hed'),
|
|
||||||
('hence'),
|
|
||||||
('her'),
|
|
||||||
('here'),
|
|
||||||
('hereafter'),
|
|
||||||
('hereby'),
|
|
||||||
('herein'),
|
|
||||||
('heres'),
|
|
||||||
('hereupon'),
|
|
||||||
('hers'),
|
|
||||||
('herself'),
|
|
||||||
('hes'),
|
|
||||||
('hi'),
|
|
||||||
('hid'),
|
|
||||||
('him'),
|
|
||||||
('himself'),
|
|
||||||
('his'),
|
|
||||||
('hither'),
|
|
||||||
('home'),
|
|
||||||
('how'),
|
|
||||||
('howbeit'),
|
|
||||||
('however'),
|
|
||||||
('hundred'),
|
|
||||||
('i'),
|
|
||||||
('id'),
|
|
||||||
('ie'),
|
|
||||||
('if'),
|
|
||||||
('i''ll'),
|
|
||||||
('im'),
|
|
||||||
('immediate'),
|
|
||||||
('immediately'),
|
|
||||||
('importance'),
|
|
||||||
('important'),
|
|
||||||
('in'),
|
|
||||||
('inc'),
|
|
||||||
('indeed'),
|
|
||||||
('index'),
|
|
||||||
('information'),
|
|
||||||
('instead'),
|
|
||||||
('into'),
|
|
||||||
('invention'),
|
|
||||||
('inward'),
|
|
||||||
('is'),
|
|
||||||
('isn''t'),
|
|
||||||
('it'),
|
|
||||||
('itd'),
|
|
||||||
('it''ll'),
|
|
||||||
('its'),
|
|
||||||
('itself'),
|
|
||||||
('i''ve'),
|
|
||||||
('j'),
|
|
||||||
('just'),
|
|
||||||
('k'),
|
|
||||||
('keep keeps'),
|
|
||||||
('kept'),
|
|
||||||
('kg'),
|
|
||||||
('km'),
|
|
||||||
('know'),
|
|
||||||
('known'),
|
|
||||||
('knows'),
|
|
||||||
('l'),
|
|
||||||
('largely'),
|
|
||||||
('last'),
|
|
||||||
('lately'),
|
|
||||||
('later'),
|
|
||||||
('latter'),
|
|
||||||
('latterly'),
|
|
||||||
('least'),
|
|
||||||
('less'),
|
|
||||||
('lest'),
|
|
||||||
('let'),
|
|
||||||
('lets'),
|
|
||||||
('like'),
|
|
||||||
('liked'),
|
|
||||||
('likely'),
|
|
||||||
('line'),
|
|
||||||
('little'),
|
|
||||||
('''ll'),
|
|
||||||
('look'),
|
|
||||||
('looking'),
|
|
||||||
('looks'),
|
|
||||||
('ltd'),
|
|
||||||
('m'),
|
|
||||||
('made'),
|
|
||||||
('mainly'),
|
|
||||||
('make'),
|
|
||||||
('makes'),
|
|
||||||
('many'),
|
|
||||||
('may'),
|
|
||||||
('maybe'),
|
|
||||||
('me'),
|
|
||||||
('mean'),
|
|
||||||
('means'),
|
|
||||||
('meantime'),
|
|
||||||
('meanwhile'),
|
|
||||||
('merely'),
|
|
||||||
('mg'),
|
|
||||||
('might'),
|
|
||||||
('million'),
|
|
||||||
('miss'),
|
|
||||||
('ml'),
|
|
||||||
('more'),
|
|
||||||
('moreover'),
|
|
||||||
('most'),
|
|
||||||
('mostly'),
|
|
||||||
('mr'),
|
|
||||||
('mrs'),
|
|
||||||
('much'),
|
|
||||||
('mug'),
|
|
||||||
('must'),
|
|
||||||
('my'),
|
|
||||||
('myself'),
|
|
||||||
('n'),
|
|
||||||
('na'),
|
|
||||||
('name'),
|
|
||||||
('namely'),
|
|
||||||
('nay'),
|
|
||||||
('nd'),
|
|
||||||
('near'),
|
|
||||||
('nearly'),
|
|
||||||
('necessarily'),
|
|
||||||
('necessary'),
|
|
||||||
('need'),
|
|
||||||
('needs'),
|
|
||||||
('neither'),
|
|
||||||
('never'),
|
|
||||||
('nevertheless'),
|
|
||||||
('new'),
|
|
||||||
('next'),
|
|
||||||
('nine'),
|
|
||||||
('ninety'),
|
|
||||||
('no'),
|
|
||||||
('nobody'),
|
|
||||||
('non'),
|
|
||||||
('none'),
|
|
||||||
('nonetheless'),
|
|
||||||
('noone'),
|
|
||||||
('nor'),
|
|
||||||
('normally'),
|
|
||||||
('nos'),
|
|
||||||
('not'),
|
|
||||||
('noted'),
|
|
||||||
('nothing'),
|
|
||||||
('now'),
|
|
||||||
('nowhere'),
|
|
||||||
('o'),
|
|
||||||
('obtain'),
|
|
||||||
('obtained'),
|
|
||||||
('obviously'),
|
|
||||||
('of'),
|
|
||||||
('off'),
|
|
||||||
('often'),
|
|
||||||
('oh'),
|
|
||||||
('ok'),
|
|
||||||
('okay'),
|
|
||||||
('old'),
|
|
||||||
('omitted'),
|
|
||||||
('on'),
|
|
||||||
('once'),
|
|
||||||
('one'),
|
|
||||||
('ones'),
|
|
||||||
('only'),
|
|
||||||
('onto'),
|
|
||||||
('or'),
|
|
||||||
('ord'),
|
|
||||||
('other'),
|
|
||||||
('others'),
|
|
||||||
('otherwise'),
|
|
||||||
('ought'),
|
|
||||||
('our'),
|
|
||||||
('ours'),
|
|
||||||
('ourselves'),
|
|
||||||
('out'),
|
|
||||||
('outside'),
|
|
||||||
('over'),
|
|
||||||
('overall'),
|
|
||||||
('owing'),
|
|
||||||
('own'),
|
|
||||||
('p'),
|
|
||||||
('page'),
|
|
||||||
('pages'),
|
|
||||||
('part'),
|
|
||||||
('particular'),
|
|
||||||
('particularly'),
|
|
||||||
('past'),
|
|
||||||
('per'),
|
|
||||||
('perhaps'),
|
|
||||||
('placed'),
|
|
||||||
('please'),
|
|
||||||
('plus'),
|
|
||||||
('poorly'),
|
|
||||||
('possible'),
|
|
||||||
('possibly'),
|
|
||||||
('potentially'),
|
|
||||||
('pp'),
|
|
||||||
('predominantly'),
|
|
||||||
('present'),
|
|
||||||
('previously'),
|
|
||||||
('primarily'),
|
|
||||||
('probably'),
|
|
||||||
('promptly'),
|
|
||||||
('proud'),
|
|
||||||
('provides'),
|
|
||||||
('put'),
|
|
||||||
('q'),
|
|
||||||
('que'),
|
|
||||||
('quickly'),
|
|
||||||
('quite'),
|
|
||||||
('qv'),
|
|
||||||
('r'),
|
|
||||||
('ran'),
|
|
||||||
('rather'),
|
|
||||||
('rd'),
|
|
||||||
('re'),
|
|
||||||
('readily'),
|
|
||||||
('really'),
|
|
||||||
('recent'),
|
|
||||||
('recently'),
|
|
||||||
('ref'),
|
|
||||||
('refs'),
|
|
||||||
('regarding'),
|
|
||||||
('regardless'),
|
|
||||||
('regards'),
|
|
||||||
('related'),
|
|
||||||
('relatively'),
|
|
||||||
('research'),
|
|
||||||
('respectively'),
|
|
||||||
('resulted'),
|
|
||||||
('resulting'),
|
|
||||||
('results'),
|
|
||||||
('right'),
|
|
||||||
('run'),
|
|
||||||
('s'),
|
|
||||||
('said'),
|
|
||||||
('same'),
|
|
||||||
('saw'),
|
|
||||||
('say'),
|
|
||||||
('saying'),
|
|
||||||
('says'),
|
|
||||||
('sec'),
|
|
||||||
('section'),
|
|
||||||
('see'),
|
|
||||||
('seeing'),
|
|
||||||
('seem'),
|
|
||||||
('seemed'),
|
|
||||||
('seeming'),
|
|
||||||
('seems'),
|
|
||||||
('seen'),
|
|
||||||
('self'),
|
|
||||||
('selves'),
|
|
||||||
('sent'),
|
|
||||||
('seven'),
|
|
||||||
('several'),
|
|
||||||
('shall'),
|
|
||||||
('she'),
|
|
||||||
('shed'),
|
|
||||||
('she''ll'),
|
|
||||||
('shes'),
|
|
||||||
('should'),
|
|
||||||
('shouldn''t'),
|
|
||||||
('show'),
|
|
||||||
('showed'),
|
|
||||||
('shown'),
|
|
||||||
('showns'),
|
|
||||||
('shows'),
|
|
||||||
('significant'),
|
|
||||||
('significantly'),
|
|
||||||
('similar'),
|
|
||||||
('similarly'),
|
|
||||||
('since'),
|
|
||||||
('six'),
|
|
||||||
('slightly'),
|
|
||||||
('so'),
|
|
||||||
('some'),
|
|
||||||
('somebody'),
|
|
||||||
('somehow'),
|
|
||||||
('someone'),
|
|
||||||
('somethan'),
|
|
||||||
('something'),
|
|
||||||
('sometime'),
|
|
||||||
('sometimes'),
|
|
||||||
('somewhat'),
|
|
||||||
('somewhere'),
|
|
||||||
('soon'),
|
|
||||||
('sorry'),
|
|
||||||
('specifically'),
|
|
||||||
('specified'),
|
|
||||||
('specify'),
|
|
||||||
('specifying'),
|
|
||||||
('still'),
|
|
||||||
('stop'),
|
|
||||||
('strongly'),
|
|
||||||
('sub'),
|
|
||||||
('substantially'),
|
|
||||||
('successfully'),
|
|
||||||
('such'),
|
|
||||||
('sufficiently'),
|
|
||||||
('suggest'),
|
|
||||||
('sup'),
|
|
||||||
('sure t'),
|
|
||||||
('take'),
|
|
||||||
('taken'),
|
|
||||||
('taking'),
|
|
||||||
('tell'),
|
|
||||||
('tends'),
|
|
||||||
('th'),
|
|
||||||
('than'),
|
|
||||||
('thank'),
|
|
||||||
('thanks'),
|
|
||||||
('thanx'),
|
|
||||||
('that'),
|
|
||||||
('that''ll'),
|
|
||||||
('thats'),
|
|
||||||
('that''ve'),
|
|
||||||
('the'),
|
|
||||||
('their'),
|
|
||||||
('theirs'),
|
|
||||||
('them'),
|
|
||||||
('themselves'),
|
|
||||||
('then'),
|
|
||||||
('thence'),
|
|
||||||
('there'),
|
|
||||||
('thereafter'),
|
|
||||||
('thereby'),
|
|
||||||
('thered'),
|
|
||||||
('therefore'),
|
|
||||||
('therein'),
|
|
||||||
('there''ll'),
|
|
||||||
('thereof'),
|
|
||||||
('therere'),
|
|
||||||
('theres'),
|
|
||||||
('thereto'),
|
|
||||||
('thereupon'),
|
|
||||||
('there''ve'),
|
|
||||||
('these'),
|
|
||||||
('they'),
|
|
||||||
('theyd'),
|
|
||||||
('they''ll'),
|
|
||||||
('theyre'),
|
|
||||||
('they''ve'),
|
|
||||||
('think'),
|
|
||||||
('this'),
|
|
||||||
('those'),
|
|
||||||
('thou'),
|
|
||||||
('though'),
|
|
||||||
('thoughh'),
|
|
||||||
('thousand'),
|
|
||||||
('throug'),
|
|
||||||
('through'),
|
|
||||||
('throughout'),
|
|
||||||
('thru'),
|
|
||||||
('thus'),
|
|
||||||
('til'),
|
|
||||||
('tip'),
|
|
||||||
('to'),
|
|
||||||
('together'),
|
|
||||||
('too'),
|
|
||||||
('took'),
|
|
||||||
('toward'),
|
|
||||||
('towards'),
|
|
||||||
('tried'),
|
|
||||||
('tries'),
|
|
||||||
('truly'),
|
|
||||||
('try'),
|
|
||||||
('trying'),
|
|
||||||
('ts'),
|
|
||||||
('twice'),
|
|
||||||
('two'),
|
|
||||||
('u'),
|
|
||||||
('un'),
|
|
||||||
('under'),
|
|
||||||
('unfortunately'),
|
|
||||||
('unless'),
|
|
||||||
('unlike'),
|
|
||||||
('unlikely'),
|
|
||||||
('until'),
|
|
||||||
('unto'),
|
|
||||||
('up'),
|
|
||||||
('upon'),
|
|
||||||
('ups'),
|
|
||||||
('us'),
|
|
||||||
('use'),
|
|
||||||
('used'),
|
|
||||||
('useful'),
|
|
||||||
('usefully'),
|
|
||||||
('usefulness'),
|
|
||||||
('uses'),
|
|
||||||
('using'),
|
|
||||||
('usually'),
|
|
||||||
('v'),
|
|
||||||
('value'),
|
|
||||||
('various'),
|
|
||||||
('''ve'),
|
|
||||||
('very'),
|
|
||||||
('via'),
|
|
||||||
('viz'),
|
|
||||||
('vol'),
|
|
||||||
('vols'),
|
|
||||||
('vs'),
|
|
||||||
('w'),
|
|
||||||
('want'),
|
|
||||||
('wants'),
|
|
||||||
('was'),
|
|
||||||
('wasnt'),
|
|
||||||
('way'),
|
|
||||||
('we'),
|
|
||||||
('wed'),
|
|
||||||
('welcome'),
|
|
||||||
('we''ll'),
|
|
||||||
('went'),
|
|
||||||
('were'),
|
|
||||||
('werent'),
|
|
||||||
('we''ve'),
|
|
||||||
('what'),
|
|
||||||
('whatever'),
|
|
||||||
('what''ll'),
|
|
||||||
('whats'),
|
|
||||||
('when'),
|
|
||||||
('whence'),
|
|
||||||
('whenever'),
|
|
||||||
('where'),
|
|
||||||
('whereafter'),
|
|
||||||
('whereas'),
|
|
||||||
('whereby'),
|
|
||||||
('wherein'),
|
|
||||||
('wheres'),
|
|
||||||
('whereupon'),
|
|
||||||
('wherever'),
|
|
||||||
('whether'),
|
|
||||||
('which'),
|
|
||||||
('while'),
|
|
||||||
('whim'),
|
|
||||||
('whither'),
|
|
||||||
('who'),
|
|
||||||
('whod'),
|
|
||||||
('whoever'),
|
|
||||||
('whole'),
|
|
||||||
('who''ll'),
|
|
||||||
('whom'),
|
|
||||||
('whomever'),
|
|
||||||
('whos'),
|
|
||||||
('whose'),
|
|
||||||
('why'),
|
|
||||||
('widely'),
|
|
||||||
('willing'),
|
|
||||||
('wish'),
|
|
||||||
('with'),
|
|
||||||
('within'),
|
|
||||||
('without'),
|
|
||||||
('wont'),
|
|
||||||
('words'),
|
|
||||||
('world'),
|
|
||||||
('would'),
|
|
||||||
('wouldnt'),
|
|
||||||
('www'),
|
|
||||||
('x'),
|
|
||||||
('y'),
|
|
||||||
('yes'),
|
|
||||||
('yet'),
|
|
||||||
('you'),
|
|
||||||
('youd'),
|
|
||||||
('you''ll'),
|
|
||||||
('your'),
|
|
||||||
('youre'),
|
|
||||||
('yours'),
|
|
||||||
('yourself'),
|
|
||||||
('yourselves'),
|
|
||||||
('you''ve'),
|
|
||||||
('z'),
|
|
||||||
('zero');
|
|
12
fedilogue/Dockerfile
Normal file
12
fedilogue/Dockerfile
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
FROM golang:latest AS build
|
||||||
|
WORKDIR /build
|
||||||
|
COPY go.mod go.sum .
|
||||||
|
RUN go mod download
|
||||||
|
COPY . .
|
||||||
|
RUN go build .
|
||||||
|
|
||||||
|
FROM alpine:latest AS run
|
||||||
|
WORKDIR /app
|
||||||
|
RUN apk add gcompat
|
||||||
|
COPY --from=build /build/fedilogue /app/fedilogue
|
||||||
|
ENTRYPOINT ["/app/fedilogue"]
|
@ -2,6 +2,8 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"flag"
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Settings - Configuration file structure
|
// Settings - Configuration file structure
|
||||||
@ -27,5 +29,9 @@ func getSettings() {
|
|||||||
flag.BoolVar(&settings.Crawl, "c", true, "Crawl mode (default is yes)")
|
flag.BoolVar(&settings.Crawl, "c", true, "Crawl mode (default is yes)")
|
||||||
flag.StringVar(&settings.Hostname, "h", "myhostname", "Set your hostname")
|
flag.StringVar(&settings.Hostname, "h", "myhostname", "Set your hostname")
|
||||||
flag.IntVar(&settings.LogLevel, "l", 1, "Logging Level:\n 0) No logs\n 1) Reports every 30 seconds\n 2) Errors\n 3) Warnings\n 4) New Connections\n 5) Debugging\n")
|
flag.IntVar(&settings.LogLevel, "l", 1, "Logging Level:\n 0) No logs\n 1) Reports every 30 seconds\n 2) Errors\n 3) Warnings\n 4) New Connections\n 5) Debugging\n")
|
||||||
|
|
||||||
|
fmt.Println("Crawl: ", settings.Crawl)
|
||||||
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
fmt.Println(settings)
|
||||||
}
|
}
|
@ -11,7 +11,7 @@ var pool *pgxpool.Pool
|
|||||||
|
|
||||||
func getDbPool() *pgxpool.Pool {
|
func getDbPool() *pgxpool.Pool {
|
||||||
// Setup Database
|
// Setup Database
|
||||||
dburl := os.Getenv("DATABASE_URL")
|
dburl := "postgres://" + os.Getenv("POSTGRES_USER") + ":" + os.Getenv("POSTGRES_PASSWORD") + "@" + os.Getenv("POSTGRES_HOSTNAME") + "/" + os.Getenv("POSTGRES_DB")
|
||||||
pool, err := pgxpool.Connect(context.Background(), dburl)
|
pool, err := pgxpool.Connect(context.Background(), dburl)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logFatal("Unable to connect to database:", err)
|
logFatal("Unable to connect to database:", err)
|
@ -1,4 +1,4 @@
|
|||||||
module git.farhan.codes/farhan/fedilogue/fedilogger
|
module git.farhan.codes/farhan/fedilogue/fedilogue
|
||||||
|
|
||||||
go 1.23.5
|
go 1.23.5
|
||||||
|
|
@ -92,6 +92,7 @@ func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningIn
|
|||||||
pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"
|
pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"
|
||||||
|
|
||||||
// Checking Mastodon and Pleroma (with .json)
|
// Checking Mastodon and Pleroma (with .json)
|
||||||
|
logDebug("GetInstanceInfo: For " + endpoint + " Mastodon/Pleroma URL: " + pleromastodon_nodeinfo_uri)
|
||||||
reqjson, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil)
|
reqjson, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil)
|
||||||
reqjson.Header.Set("User-Agent", "Tusky")
|
reqjson.Header.Set("User-Agent", "Tusky")
|
||||||
|
|
||||||
@ -116,6 +117,7 @@ func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningIn
|
|||||||
|
|
||||||
// Checking for Misskey (without .json)
|
// Checking for Misskey (without .json)
|
||||||
misskey_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0"
|
misskey_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0"
|
||||||
|
logDebug("GetInstanceInfo: For " + endpoint + " MissKey URL: " + misskey_nodeinfo_uri)
|
||||||
req, _ := http.NewRequest("GET", misskey_nodeinfo_uri, nil)
|
req, _ := http.NewRequest("GET", misskey_nodeinfo_uri, nil)
|
||||||
req.Header.Set("User-Agent", "Tusky")
|
req.Header.Set("User-Agent", "Tusky")
|
||||||
|
|
||||||
@ -140,6 +142,7 @@ func GetInstanceInfo(endpoint string, o shared.RunningInstance) shared.RunningIn
|
|||||||
|
|
||||||
// Check for Lemmy (With Json)
|
// Check for Lemmy (With Json)
|
||||||
lemmy_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.1"
|
lemmy_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.1"
|
||||||
|
logDebug("GetInstanceInfo: For " + endpoint + " Lemmy URL: " + lemmy_nodeinfo_uri)
|
||||||
req, _ = http.NewRequest("GET", lemmy_nodeinfo_uri, nil)
|
req, _ = http.NewRequest("GET", lemmy_nodeinfo_uri, nil)
|
||||||
req.Header.Set("User-Agent", "Tusky")
|
req.Header.Set("User-Agent", "Tusky")
|
||||||
|
|
||||||
@ -227,7 +230,10 @@ func CheckInstance(newinstance string, callerEndpoint string) {
|
|||||||
go StartInstance(newinstance)
|
go StartInstance(newinstance)
|
||||||
}
|
}
|
||||||
ri_mutex.Unlock()
|
ri_mutex.Unlock()
|
||||||
|
} else {
|
||||||
|
logDebug("Not crawling into ", newinstance)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func staggeredStart() {
|
func staggeredStart() {
|
||||||
@ -257,7 +263,6 @@ func StartInstance(endpoint string) {
|
|||||||
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
|
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
|
||||||
o.CaptureType = "Stream"
|
o.CaptureType = "Stream"
|
||||||
UpdateRunner(endpoint, o)
|
UpdateRunner(endpoint, o)
|
||||||
// PollMastodonPleroma(endpoint, &o)
|
|
||||||
StreamPleroma(endpoint)
|
StreamPleroma(endpoint)
|
||||||
} else if o.Software == "mastodon" {
|
} else if o.Software == "mastodon" {
|
||||||
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
|
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
|
@ -45,7 +45,7 @@ type MisskeyRequestParams struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func StreamMisskey(endpoint string) {
|
func StreamMisskey(endpoint string) {
|
||||||
logDebug("StreamMisskey for ", endpoint)
|
logDebug("Stream Misskey for ", endpoint)
|
||||||
u := url.URL{
|
u := url.URL{
|
||||||
Scheme: "wss",
|
Scheme: "wss",
|
||||||
Host: endpoint,
|
Host: endpoint,
|
||||||
@ -200,6 +200,7 @@ func StreamPleroma(endpoint string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func StreamMastodon(endpoint string, o *shared.RunningInstance) {
|
func StreamMastodon(endpoint string, o *shared.RunningInstance) {
|
||||||
|
logDebug("Stream Mastodon for ", endpoint)
|
||||||
stream_client := BuildClient(endpoint)
|
stream_client := BuildClient(endpoint)
|
||||||
|
|
||||||
var retry bool
|
var retry bool
|
||||||
@ -207,6 +208,7 @@ func StreamMastodon(endpoint string, o *shared.RunningInstance) {
|
|||||||
api_timeline := "https://" + endpoint + "/api/v1/streaming/public"
|
api_timeline := "https://" + endpoint + "/api/v1/streaming/public"
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
logDebug("StreamMastodon: " + endpoint + " URL: " + api_timeline)
|
||||||
req, err := http.NewRequest("GET", api_timeline, nil)
|
req, err := http.NewRequest("GET", api_timeline, nil)
|
||||||
req.Header.Set("User-Agent", "Tusky")
|
req.Header.Set("User-Agent", "Tusky")
|
||||||
if err != nil {
|
if err != nil {
|
@ -5,7 +5,8 @@ RUN go mod download
|
|||||||
COPY . .
|
COPY . .
|
||||||
RUN go build .
|
RUN go build .
|
||||||
|
|
||||||
FROM alpine:latest
|
FROM alpine:latest AS run
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
RUN apk add gcompat
|
||||||
COPY --from=build /build/restapi /app/restapi
|
COPY --from=build /build/restapi /app/restapi
|
||||||
ENTRYPOINT ["/app/restapi"]
|
ENTRYPOINT ["/app/restapi"]
|
||||||
|
@ -11,7 +11,8 @@ var pool *pgxpool.Pool
|
|||||||
|
|
||||||
func getDbPool() *pgxpool.Pool {
|
func getDbPool() *pgxpool.Pool {
|
||||||
// Setup Database
|
// Setup Database
|
||||||
pool, err := pgxpool.Connect(context.Background(), os.Getenv("DATABASE_URL"))
|
dburl := "postgres://" + os.Getenv("POSTGRES_USER") + ":" + os.Getenv("POSTGRES_PASSWORD") + "@" + os.Getenv("POSTGRES_HOSTNAME") + "/" + os.Getenv("POSTGRES_DB")
|
||||||
|
pool, err := pgxpool.Connect(context.Background(), dburl)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal("Unable to connect to database:", err)
|
log.Fatal("Unable to connect to database:", err)
|
||||||
}
|
}
|
||||||
|
@ -1,5 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
|
|
||||||
sleep 5
|
|
||||||
|
|
||||||
./restapi
|
|
@ -383,7 +383,7 @@ func getGlobalGraph(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
line.SetXAxis(dates).
|
line.SetXAxis(dates).
|
||||||
AddSeries("Post Count", generateLineItems(counts)).
|
AddSeries("Post Count", generateLineItems(counts)).
|
||||||
SetSeriesOptions(charts.WithLineChartOpts(opts.LineChart{Smooth: false}))
|
SetSeriesOptions(charts.WithLineChartOpts(opts.LineChart{Smooth: opts.Bool(false)}))
|
||||||
|
|
||||||
line.Render(w)
|
line.Render(w)
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user