farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding LogLevel setting

Browse Source
This commit is contained in:
farhan 2021-02-11 21:05:01 +00:00
parent d6b03c5254
commit 947645828e
10 changed files with 103 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fedilogue/config.go
View File

@ -50,6 +50,7 @@ type Settings struct {
Externalaccounts []ExtAccount `"json:externalaccounts"`
MassFollowers []MassFollower `"json:massfollowers"`
Database Database `"json:database"`
LogLevel int `"json:loglevel"`
}
var settings Settings

13
fedilogue/config.jsonc.sample
View File

@ -87,5 +87,16 @@
"username": "fedilogue",
"password": "",
"workers": 4
}
},
/*
Log Level:
* 0 = No logs
* 1 = Reports every 30 seconds
* 2 = Errors
* 3 = Warnings
* 4 = New Connections
* 5 = Debugging
*/
"loglevel": 1
}

8
fedilogue/ctl.go
View File

@ -8,7 +8,7 @@ import (
)
func startctl() {
logInfo.Print("Starting ctl listener on 127.0.0.1:5555")
logInfo("Starting ctl listener on 127.0.0.1:5555")
l, err := net.Listen("tcp", "127.0.0.1:5555")
if err != nil {
logFatal.Fatal("Unable to start listener:", err)
@ -67,11 +67,11 @@ func handleClient(commandClient net.Conn) {
case "status":
responseback.Message = "Ok"
case "add":
logInfo.Print("Manually added instance: " + commandmap.Endpoint)
logInfo("Manually added instance: " + commandmap.Endpoint)
ri_mutex.Lock()
_, exists := runninginstances[commandmap.Endpoint]
if exists == true {
logInfo.Println("Already exists: " + commandmap.Endpoint)
logInfo("Already exists: " + commandmap.Endpoint)
responseback.Message = "Exists: " + commandmap.Endpoint
} else {
responseback.Message = "Adding: " + commandmap.Endpoint
@ -96,7 +96,7 @@ func handleClient(commandClient net.Conn) {
responsebytes, err := json.Marshal(responseback)
if err != nil {
logErr.Fatal(err)
logFatal.Fatal(err)
}
n = len(responsebytes)

6
fedilogue/fedilogue.go
View File

@ -16,7 +16,7 @@ var ri_mutex = &sync.Mutex{}
var pool *pgxpool.Pool
func startpprof() {
logInfo.Print("Starting http/pprof on :7777")
logInfo("Starting http/pprof on :7777")
logFatal.Fatal(http.ListenAndServe("127.0.0.1:7777", nil))
}
@ -28,7 +28,7 @@ func main() {
getSettings()
if len(settings.Proxies) > 0 {
for i := 0; i < len(settings.Proxies); i++ {
logInfo.Printf("Using proxy: %s:%d", settings.Proxies[i].Host, settings.Proxies[i].Port)
logInfo("Using proxy: ", settings.Proxies[i].Host, ":", settings.Proxies[i].Port)
}
}
go startpprof()
@ -42,7 +42,7 @@ func main() {
matchurl = regexp.MustCompile("http?s://[\\w\\-]+\\.[\\w\\-]+\\S*")
for _, endpoint := range settings.Autostart {
logInfo.Print("Autostarting " + endpoint)
logInfo("Autostarting " + endpoint)
_, exists := GetRunner(endpoint)
if exists == false {
go StartInstance(endpoint)

16
fedilogue/instance.go
View File

@ -20,7 +20,7 @@ func DoTries(o *RunningInstance, req *http.Request) (*http.Response, error) {
resp, err = o.client.Do(req)
if err != nil {
// URL.Scheme, Host, Path Opaque
logWarn.Print("Failure connecting to " + req.URL.Scheme + "://" + req.URL.Host + req.URL.Path + ", attempt ", tries + 1, ", sleeping for 5 minutes: ", err)
logWarn("Failure connecting to " + req.URL.Scheme + "://" + req.URL.Host + req.URL.Path + ", attempt ", tries + 1, ", sleeping for 5 minutes: ", err)
time.Sleep(time.Minute * 5)
continue
}
@ -115,7 +115,7 @@ func GetNodeInfo(endpoint string, o RunningInstance) RunningInstance {
o.LastRun = time.Now().Format(time.RFC3339)
if err != nil {
o.Status = UNSUPPORTED_INSTANCE
logWarn.Print("Unable to connect to " + endpoint + ", giving up")
logWarn("Unable to connect to " + endpoint + ", giving up")
return o
}
defer resp_index.Body.Close()
@ -123,7 +123,7 @@ func GetNodeInfo(endpoint string, o RunningInstance) RunningInstance {
indexbin, err := ioutil.ReadAll(resp_index.Body)
if err != nil {
o.Status = UNSUPPORTED_INSTANCE
logWarn.Print("Unable to read index of " + endpoint + ", giving up")
logWarn("Unable to read index of " + endpoint + ", giving up")
return o
}
indexstr := string(indexbin)
@ -153,14 +153,14 @@ func CheckInstance(newinstance string, callerEndpoint string) {
for attempt := 0; attempt > 5; attempt = attempt + 1 {
_, err = net.LookupHost(newinstance)
if err != nil {
logDebug.Print("Unable to resolve " + newinstance + " attempt ", attempt, "/5. Sleeping for 30 seconds")
logDebug("Unable to resolve " + newinstance + " attempt ", attempt, "/5. Sleeping for 30 seconds")
time.Sleep(time.Second * 30)
continue
}
break
}
if err != nil {
logWarn.Print("Unable to resolve ", newinstance, " after 5 attempts, giving up: ", err)
logWarn("Unable to resolve ", newinstance, " after 5 attempts, giving up: ", err)
return
}
@ -197,16 +197,16 @@ func StartInstance(endpoint string) {
}
if o.Software == "pleroma" {
logInfo.Print("Starting " + endpoint + " as " + o.Software + " " + o.Version)
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
o.CaptureType = "Poll"
UpdateRunner(endpoint, o)
PollMastodonPleroma(endpoint, &o)
} else if o.Software == "mastodon" {
logInfo.Print("Starting " + endpoint + " as " + o.Software + " " + o.Version)
logConn("Starting " + endpoint + " as " + o.Software + " " + o.Version)
o.CaptureType = "Stream"
UpdateRunner(endpoint, o)
StreamMastodon(endpoint, &o)
} else {
logWarn.Print("Unsupported endpoint " + endpoint)
logConn("Unsupported endpoint " + endpoint)
}
}

50
fedilogue/log.go
View File

@ -6,17 +6,49 @@ import (
)
var (
logInfo *log.Logger
logWarn *log.Logger
logErr *log.Logger
logDebug *log.Logger
logFatal *log.Logger
_logInfo *log.Logger
_logErr *log.Logger
_logWarn *log.Logger
_logConn *log.Logger
_logDebug *log.Logger
logFatal *log.Logger
)
func logInit() {
logInfo = log.New(os.Stdout, "INFO: ", log.Lmsgprefix|log.Ldate|log.Ltime)
logWarn = log.New(os.Stdout, "WARN: ", log.Lmsgprefix|log.Ldate|log.Ltime)
logErr = log.New(os.Stdout, "ERR: ", log.Lmsgprefix|log.Ldate|log.Ltime)
_logInfo = log.New(os.Stdout, "INFO: ", log.Lmsgprefix|log.Ldate|log.Ltime)
_logErr = log.New(os.Stdout, "ERR: ", log.Lmsgprefix|log.Ldate|log.Ltime)
_logWarn = log.New(os.Stdout, "WARN: ", log.Lmsgprefix|log.Ldate|log.Ltime)
_logConn = log.New(os.Stdout, "CONN: ", log.Lmsgprefix|log.Ldate|log.Ltime)
_logDebug = log.New(os.Stdout, "DEBUG: ", log.Lmsgprefix|log.Ldate|log.Ltime)
logFatal = log.New(os.Stdout, "FATAL: ", log.Lmsgprefix|log.Ldate|log.Ltime)
logDebug = log.New(os.Stdout, "DEBUG: ", log.Lmsgprefix|log.Ldate|log.Ltime)
}
func logInfo(s ...interface{}) {
if (settings.LogLevel >= 1) {
_logInfo.Print(s...)
}
}
func logErr(s ...interface{}) {
if (settings.LogLevel >= 2) {
_logErr.Print(s...)
}
}
func logWarn(s ...interface{}) {
if (settings.LogLevel >= 3) {
_logWarn.Print(s...)
}
}
func logConn(s ...interface{}) {
if (settings.LogLevel >= 4) {
_logConn.Print(s...)
}
}
func logDebug(s ...interface{}) {
if (settings.LogLevel >= 5) {
_logConn.Print(s...)
}
}

34
fedilogue/oauth.go
View File

@ -37,14 +37,14 @@ func register_client(endpoint string, o *RunningInstance) error {
resp, err := o.client.Post(api_base_apps, "application/json", requestBodybytes)
if err != nil {
logErr.Print("Unable to connect to "+api_base_apps+" ", err)
logErr("Unable to connect to "+api_base_apps+" ", err)
return err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logErr.Print("Unable to read HTTP response: ", err)
logErr("Unable to read HTTP response: ", err)
o.client_id = ""
o.client_secret = ""
return err
@ -53,7 +53,7 @@ func register_client(endpoint string, o *RunningInstance) error {
bodymap := make(map[string]string)
err = json.Unmarshal(body, &bodymap)
if err != nil {
logErr.Print("Unable to parse response from "+endpoint+": ", err)
logErr("Unable to parse response from "+endpoint+": ", err)
o.client_id = ""
o.client_secret = ""
return err
@ -63,7 +63,7 @@ func register_client(endpoint string, o *RunningInstance) error {
f, err := os.Create("clients/" + endpoint)
if err != nil {
logErr.Print("Unable to create "+client_file+": ", err)
logErr("Unable to create "+client_file+": ", err)
o.client_id = ""
o.client_secret = ""
return err
@ -72,14 +72,14 @@ func register_client(endpoint string, o *RunningInstance) error {
_, err = io.WriteString(f, bodymap["client_id"]+"\n")
if err != nil {
logErr.Print("Unable to write client_id line to file "+client_file+": ", err)
logErr("Unable to write client_id line to file "+client_file+": ", err)
o.client_id = bodymap["client_id"]
o.client_secret = bodymap["client_secret"]
return nil
}
_, err = io.WriteString(f, bodymap["client_secret"]+"\n")
if err != nil {
logErr.Print("Unable to write client_secret to file "+client_file+": ", err)
logErr("Unable to write client_secret to file "+client_file+": ", err)
o.client_id = bodymap["client_id"]
o.client_secret = bodymap["client_secret"]
return nil
@ -97,7 +97,7 @@ func get_client(endpoint string, o *RunningInstance) error {
if os.IsNotExist(err) == false { // The file exists
f, err := os.Open(client_file)
if err != nil {
logErr.Print("Unable to open " + client_file + ", creating new client")
logErr("Unable to open " + client_file + ", creating new client")
return err
}
defer f.Close()
@ -107,13 +107,13 @@ func get_client(endpoint string, o *RunningInstance) error {
client_id_bin, _, err := rd.ReadLine()
o.client_id = string(client_id_bin)
if err != nil {
logErr.Print("Unable to read client_id line of " + client_file + ", building new client")
logErr("Unable to read client_id line of "+client_file+", building new client")
return err
}
client_secret_bin, _, err := rd.ReadLine()
o.client_secret = string(client_secret_bin)
if err != nil {
logErr.Print("Unable to read client_secret line of " + client_file + ", building new client")
logErr("Unable to read client_secret line of "+client_file+", building new client")
return err
}
@ -138,7 +138,7 @@ func oauth_login(endpoint string, o *RunningInstance, username string, password
})
if err != nil {
logErr.Print("Unable to create Authentication map for "+endpoint)
logErr("Unable to create Authentication map for "+endpoint)
return OAuth{}, err
}
@ -146,26 +146,26 @@ func oauth_login(endpoint string, o *RunningInstance, username string, password
resp, err := http.Post("https://"+endpoint+"/oauth/token", "application/json", authMapbytes)
if err != nil {
logErr.Print("Cannot connect to "+endpoint+": ", err)
logErr("Cannot connect to "+endpoint+": ", err)
return OAuth{}, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logErr.Print("Unable to read response data for "+endpoint+": ", err)
logErr("Unable to read response data for "+endpoint+": ", err)
return OAuth{}, err
}
if resp.StatusCode == 400 {
logErr.Print("Unable to authenticate to " + endpoint)
logErr("Unable to authenticate to " + endpoint)
return OAuth{}, &authError{"Authentication error"}
}
oauthData := OAuth{}
err = json.Unmarshal(body, &oauthData)
if err != nil {
logErr.Print("Unable to parse json data for "+endpoint+": ", err)
logErr("Unable to parse json data for "+endpoint+": ", err)
return OAuth{}, err
}
@ -186,21 +186,21 @@ func oauth_refresh(endpoint string, client_id string, client_secret string, refr
resp, err := http.Post("https://"+endpoint+"/oauth/token", "application/json", authMapbytes)
if err != nil {
logErr.Print("Unable to connect to "+endpoint+": ", err)
logErr("Unable to connect to "+endpoint+": ", err)
return OAuth{}, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logErr.Print("Unable to read response data for "+endpoint+": ", err)
logErr("Unable to read response data for "+endpoint+": ", err)
return OAuth{}, err
}
oauthData := OAuth{}
err = json.Unmarshal(body, &oauthData)
if err != nil {
logErr.Print("Unable to parse json data for "+endpoint+": ", err)
logErr("Unable to parse json data for "+endpoint+": ", err)
return oauthData, err
}

16
fedilogue/poll.go
View File

@ -66,13 +66,13 @@ func PollMastodonPleroma(endpoint string, o *RunningInstance) {
o, _ := GetRunner(endpoint)
err = get_client(endpoint, &o)
if err != nil {
logErr.Print("Unable to register client for " + endpoint + ": ", err)
logErr("Unable to register client for " + endpoint + ": ", err)
return
}
oauthData, err = oauth_login(endpoint, &o, extaccount.Username, extaccount.Password)
if err != nil {
logErr.Print("Unable to login to " + endpoint + ": ", err)
logErr("Unable to login to " + endpoint + ": ", err)
return
}
last_refresh = time.Now().Unix()
@ -97,7 +97,7 @@ func PollMastodonPleroma(endpoint string, o *RunningInstance) {
if time.Now().Unix() > last_refresh+oauthData.Expires_in {
oauthData, err = oauth_refresh(endpoint, client_id, client_secret, oauthData.Refresh_token)
if err != nil {
logWarn.Fatal("Unable to refresh oauth token for " + endpoint + ": ", err)
logWarn("Unable to refresh oauth token for " + endpoint + ": ", err)
return
}
last_refresh = time.Now().Unix()
@ -112,12 +112,12 @@ func PollMastodonPleroma(endpoint string, o *RunningInstance) {
ri_mutex.Lock()
runninginstances[endpoint] = m
ri_mutex.Unlock()
logWarn.Print("Giving up on " + endpoint + ": ", err.Error())
logWarn("Giving up on " + endpoint + ": ", err.Error())
return
}
if resp.StatusCode == TOOMANYREQUESTS { // Short Delay, 30 seconds
logWarn.Print("Delaying "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay")
logWarn("Delaying "+endpoint+", gave status ", resp.StatusCode, ", 1 hour delay")
_, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close() // Release as soon as done
m.Status = resp.StatusCode
@ -127,7 +127,7 @@ func PollMastodonPleroma(endpoint string, o *RunningInstance) {
time.Sleep(time.Second * 30)
continue
} else if resp.StatusCode == INTERNAL_ERROR { // Longer delay, 1 hour
logWarn.Print("Suspending " + endpoint + ", gave status ", resp.StatusCode, ", 1 hour delay")
logWarn("Suspending " + endpoint + ", gave status ", resp.StatusCode, ", 1 hour delay")
_, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close() // Release as soon as done
m.Status = 765
@ -137,7 +137,7 @@ func PollMastodonPleroma(endpoint string, o *RunningInstance) {
time.Sleep(time.Second * 3600)
continue
} else if resp.StatusCode != 200 { // Crash
logErr.Print("Terminating " + endpoint + ", gave status ", resp.StatusCode)
logErr("Terminating " + endpoint + ", gave status ", resp.StatusCode)
_, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close() // Release as soon as done
m.Status = resp.StatusCode
@ -155,7 +155,7 @@ func PollMastodonPleroma(endpoint string, o *RunningInstance) {
ri_mutex.Lock()
runninginstances[endpoint] = m
ri_mutex.Unlock()
logErr.Print("Giving up on " + endpoint + " after 5 unmarshal errors.")
logErr("Giving up on " + endpoint + " after 5 unmarshal errors.")
return
}
parsing_error = parsing_error + 1

8
fedilogue/retrieve.go
View File

@ -155,7 +155,7 @@ func check_activity(uri string) {
_, err = pool.Exec(context.Background(), "INSERT INTO activities (document, normalized, instance) VALUES($1, $2, $3)", jsondocument, activityjson.normalized, activityjson.instance)
if err != nil {
logWarn.Printf("Error inserting %s into `activities`: %s", uri, err)
logWarn("Error inserting %s into `activities`: "+ uri, err)
//return activityjson, err
return
}
@ -217,11 +217,11 @@ func check_actor(uri string) {
resp, err = o.client.Do(req)
if err != nil {
if tries > 10 {
logErr.Print("Unable to connect to "+uri+" attempt 10/10, giving up.")
logErr("Unable to connect to "+uri+" attempt 10/10, giving up.")
// return actorjson, err
return
}
logWarn.Print("Unable to connect to "+uri+", attempt ",tries+1,"+/10 sleeping for 30 seconds.")
logWarn("Unable to connect to "+uri+", attempt ",tries+1,"+/10 sleeping for 30 seconds.")
time.Sleep(time.Second * 30)
tries = tries + 1
continue
@ -246,7 +246,7 @@ func check_actor(uri string) {
_, err = pool.Exec(context.Background(), "INSERT INTO actors (document, instance) VALUES($1, $2)", jsondocument, actorjson.instance)
if err != nil {
logWarn.Printf("Error inserting %s into `actors`: %s", uri, err)
logWarn("Error inserting %s into `actors`: "+uri, err)
// return actorjson, err
return
}

10
fedilogue/stream.go
View File

@ -31,12 +31,12 @@ func StreamMastodon(endpoint string, o *RunningInstance) {
err = get_client(endpoint, o)
if err != nil {
logWarn.Print("Unable to register client: ", err)
logWarn("Unable to register client: ", err)
}
oauthData, err = oauth_login(endpoint, o, extaccount.Username, extaccount.Password)
if err != nil {
logWarn.Print("Unable to login: ", err)
logWarn("Unable to login: ", err)
return
}
@ -51,13 +51,13 @@ func StreamMastodon(endpoint string, o *RunningInstance) {
resp, err = stream_client.Do(req)
if err != nil {
time.Sleep(time.Minute * 5)
logWarn.Print("Failure connecting to " + req.URL.Scheme + "://" + req.URL.Host + req.URL.Path + ", attempt ", tries + 1, ", sleeping for 5 minutes, ", err)
logWarn("Failure connecting to " + req.URL.Scheme + "://" + req.URL.Host + req.URL.Path + ", attempt ", tries + 1, ", sleeping for 5 minutes, ", err)
continue
}
break
}
if err != nil {
logErr.Print("Unable to stream " + api_timeline + ": ", err)
logErr("Unable to stream " + api_timeline + ": ", err)
return
}
defer resp.Body.Close()
@ -89,7 +89,7 @@ func StreamMastodon(endpoint string, o *RunningInstance) {
jsondata := token[1][1:]
err := json.Unmarshal([]byte(jsondata), &newactivity)
if err != nil {
logDebug.Print("Unable to parse data from "+endpoint+", but still connected.")
logDebug("Unable to parse data from "+endpoint+", but still connected.")
continue
}
retry = true