farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reduced the number of new connections to 1, reduce spamming servers

Browse Source 
set keep-alive to 2 hours
probably a bunch of regressions but I don't have unit tests yet
This commit is contained in:
farhan 2020-12-25 05:18:42 +00:00
parent 9db3c05ed6
commit 88c074f76b
5 changed files with 129 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
headers.go
View File

@ -49,10 +49,13 @@ type AccountType struct {
// Instance's new min_id value
type RunningInstance struct {
Software string `json:"software"`
Version string `json:"version"`
Status int `json:"status"`
LastRun string `json:"lastrun"`
CaptureType string `json:"capturetype"`
client http.Client
client_id string
client_secret string
}
type NodeInfoSoftware struct {

113
instance.go
View File

@ -14,92 +14,115 @@ import (
var p *bluemonday.Policy
var spaceReg *regexp.Regexp
// Change this to return a proper "err"
func GetNodeInfo(endpoint string) (http.Client, NodeInfo) {
func GetRunner(endpoint string) (RunningInstance) {
ri_mutex.Lock()
o, exists := runninginstances[endpoint]
if exists == false {
o := RunningInstance{}
tr := &http.Transport {MaxIdleConns: 10, IdleConnTimeout: 7200 * time.Second}
o.client = http.Client{Transport: tr}
o.Status = KEEPALIVE
runninginstances[endpoint] = o
}
ri_mutex.Unlock()
return o
}
func UpdateRunner(endpoint string, o RunningInstance) {
ri_mutex.Lock()
runninginstances[endpoint] = o
ri_mutex.Unlock()
}
func GetNodeInfo(endpoint string, o RunningInstance) (RunningInstance) {
/* Checking order
* Mastodon/Pleroma
* Um..nothing else yet
*/
var nodeinfo NodeInfo
pleromastodon_nodeinfo_uri := "https://" + endpoint + "/nodeinfo/2.0.json"
http_client := http.Client{}
pleromastodon_api_resp, err := http_client.Get(pleromastodon_nodeinfo_uri)
req, _ := http.NewRequest("GET", pleromastodon_nodeinfo_uri, nil)
pleromastodon_api_resp, err := o.client.Do(req)
if err != nil {
return http_client, NodeInfo{}
return o
} else {
defer pleromastodon_api_resp.Body.Close()
}
if pleromastodon_api_resp.StatusCode == 200 {
var nodeinfo NodeInfo
err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo)
if err == nil {
o.Software = nodeinfo.Software.Name
o.LastRun = time.Now().Format(time.RFC3339)
defer pleromastodon_api_resp.Body.Close()
return http_client, nodeinfo
return o
}
}
// Check the front page
index_uri := "https://" + endpoint + "/"
resp_index, err := http_client.Get(index_uri)
req, _ = http.NewRequest("GET", index_uri, nil)
resp_index, err := o.client.Do(req)
o.LastRun = time.Now().Format(time.RFC3339)
if err != nil {
o.Status = UNSUPPORTED_INSTANCE
log.Print("Unable to connect to " + endpoint + ", giving up")
return http_client, NodeInfo{}
return o
}
defer resp_index.Body.Close()
indexbin, err := ioutil.ReadAll(resp_index.Body)
if err != nil {
o.Status = UNSUPPORTED_INSTANCE
log.Print("Unable to read index of " + endpoint + ", giving up")
return http_client, NodeInfo{}
return o
}
indexstr := string(indexbin)
nodeinfo := NodeInfo{}
if strings.Contains(indexstr, "Pleroma") || strings.Contains(indexstr, "Soapbox") {
log.Print("Manual view: Pleroma" + endpoint)
nodeinfo.Software.Name = "pleroma"
nodeinfo.Software.Version = "guess"
o.Software = "pleroma"
o.Version = "guess"
} else if strings.Contains(indexstr, "Mastodon") {
log.Print("Manual view: Mastodon" + endpoint)
nodeinfo.Software.Name = "mastodon"
nodeinfo.Software.Version = "guess"
o.Software = "mastodon"
o.Version = "guess"
} else if strings.Contains(indexstr, "Gab") {
log.Print("Manual view: Gab" + endpoint)
nodeinfo.Software.Name = "gab"
nodeinfo.Software.Version = "guess"
o.Software = "gab"
o.Version = "guess"
}
return http_client, nodeinfo
return o
}
func StartInstance(endpoint string) {
http_client, nodeinfo := GetNodeInfo(endpoint)
ri_mutex.Lock()
m := runninginstances[endpoint]
ri_mutex.Unlock()
if nodeinfo.Software.Name == "" {
m.Software = ""
m.LastRun = time.Now().Format(time.RFC3339)
m.Status = UNSUPPORTED_INSTANCE
ri_mutex.Lock()
runninginstances[endpoint] = m
ri_mutex.Unlock()
// Check if exists. If so, get the object. If not, create it
o := GetRunner(endpoint)
o = GetNodeInfo(endpoint, o)
UpdateRunner(endpoint, o)
if o.Software == "" {
return
}
m.client = http_client
if nodeinfo.Software.Name == "pleroma" {
log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
m.CaptureType = "Poll"
ri_mutex.Lock()
runninginstances[endpoint] = m
ri_mutex.Unlock()
PollMastodonPleroma(endpoint, http_client)
} else if nodeinfo.Software.Name == "mastodon" {
log.Print("Starting " + endpoint + " as " + nodeinfo.Software.Name)
m.CaptureType = "Stream"
ri_mutex.Lock()
runninginstances[endpoint] = m
ri_mutex.Unlock()
StreamMastodon(endpoint)
if o.Software == "pleroma" {
log.Print("Starting " + endpoint + " as " + o.Software)
o.CaptureType = "Poll"
UpdateRunner(endpoint, o)
PollMastodonPleroma(endpoint, &o)
} else if o.Software == "mastodon" {
log.Print("Starting " + endpoint + " as " + o.Software)
o.CaptureType = "Stream"
UpdateRunner(endpoint, o)
StreamMastodon(endpoint, &o)
}
}

72
oauth.go
View File

@ -26,7 +26,7 @@ func (e *authError) Error() string {
return e.msg
}
func register_client(endpoint string, http_client *http.Client) (string, string, error) {
func register_client(endpoint string, o *RunningInstance) (error) {
requestBodymap, _ := json.Marshal(map[string]string{
"client_name": "Tusky", // Hard-coded in for now...
"scopes": "read write follow push",
@ -36,79 +36,101 @@ func register_client(endpoint string, http_client *http.Client) (string, string,
api_base_apps := "https://" + endpoint + "/api/v1/apps"
resp, err := http_client.Post(api_base_apps, "application/json", requestBodybytes)
resp, err := o.client.Post(api_base_apps, "application/json", requestBodybytes)
if err != nil {
log.Fatal("Unable to connect to "+api_base_apps+" ", err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Print("Unable to read HTTP response: ", err)
return "", "", err
log.Fatal("Unable to read HTTP response: ", err)
o.client_id = ""
o.client_secret = ""
return err
}
defer resp.Body.Close()
bodymap := make(map[string]string)
err = json.Unmarshal(body, &bodymap)
if err != nil {
log.Print("Unable to Unmarshal response: ", err)
return "", "", err
log.Fatal("Unable to Unmarshal response: ", err)
o.client_id = ""
o.client_secret = ""
return err
}
client_file := "clients/" + endpoint
f, err := os.Create("clients/" + endpoint)
if err != nil {
log.Print("Unable to create "+client_file+": ", err)
return bodymap["client_id"], bodymap["client_secret"], nil
log.Fatal("Unable to create "+client_file+": ", err)
o.client_id = ""
o.client_secret = ""
return err
//return bodymap["client_id"], bodymap["client_secret"], nil
}
defer f.Close()
_, err = io.WriteString(f, bodymap["client_id"]+"\n")
if err != nil {
log.Print("Unable to write client_id line: ", err)
return bodymap["client_id"], bodymap["client_secret"], nil
log.Fatal("Unable to write client_id line: ", err)
o.client_id = bodymap["client_id"]
o.client_secret = bodymap["client_secret"]
return nil
}
_, err = io.WriteString(f, bodymap["client_secret"]+"\n")
if err != nil {
log.Print("Unable to write client_secret line: ", err)
return bodymap["client_id"], bodymap["client_secret"], nil
log.Fatal("Unable to write client_secret line: ", err)
o.client_id = bodymap["client_id"]
o.client_secret = bodymap["client_secret"]
return nil
}
return bodymap["client_id"], bodymap["client_secret"], nil
o.client_id = bodymap["client_id"]
o.client_secret = bodymap["client_secret"]
return nil
}
func get_client(endpoint string, http_client *http.Client) (string, string, error) {
func get_client(endpoint string, o *RunningInstance) (error) {
var err error
client_file := "clients/" + endpoint
_, err := os.Stat(client_file)
_, err = os.Stat(client_file)
if os.IsNotExist(err) == false { // The file exists
f, err := os.Open(client_file)
if err != nil {
log.Print("Unable to open " + client_file + ", creating new client")
return register_client(endpoint, http_client)
return err
// return register_client(endpoint, o)
}
defer f.Close()
rd := bufio.NewReader(f)
client_id, _, err := rd.ReadLine()
client_id_bin, _, err := rd.ReadLine()
o.client_id = string(client_id_bin)
if err != nil {
log.Print("Unable to read client_id line of " + client_file + ", building new client")
return register_client(endpoint, http_client)
return err
// return register_client(endpoint, o)
}
client_secret, _, err := rd.ReadLine()
client_secret_bin, _, err := rd.ReadLine()
o.client_secret = string(client_secret_bin)
if err != nil {
log.Print("Unable to read client_secret line of " + client_file + ", building new client")
return register_client(endpoint, http_client)
return err
// return register_client(endpoint, o)
// return o
}
return string(client_id), string(client_secret), nil
return nil
} else {
return register_client(endpoint, http_client)
return register_client(endpoint, o)
}
return nil
}
func oauth_login(endpoint string, username string, password string, client_id string, client_secret string) (OAuth, error) {
func oauth_login(endpoint string, o *RunningInstance, username string, password string) (OAuth, error) {
authMap, err := json.Marshal(map[string]string{
"username": username,
"password": password,
@ -116,8 +138,8 @@ func oauth_login(endpoint string, username string, password string, client_id st
"grant_type": "password",
"client_name": "Tusky",
"scope": "read write follow push",
"client_id": client_id,
"client_secret": client_secret,
"client_id": o.client_id,
"client_secret": o.client_secret,
})
if err != nil {

9
poll.go
View File

@ -50,7 +50,7 @@ type PostInfo struct {
Content string `"json:content"`
}
func PollMastodonPleroma(endpoint string, http_client http.Client) {
func PollMastodonPleroma(endpoint string, o *RunningInstance) {
newposts := make([]ReportPost, 0)
min_id := ""
@ -68,13 +68,14 @@ func PollMastodonPleroma(endpoint string, http_client http.Client) {
for _, extaccount := range settings.Externalaccounts {
if extaccount.Endpoint == endpoint {
use_auth = true
client_id, client_secret, err = get_client(endpoint, &http_client)
o := GetRunner(endpoint)
err = get_client(endpoint, &o)
if err != nil {
log.Print("Unable to register client: ", err)
return
}
oauthData, err = oauth_login(endpoint, extaccount.Username, extaccount.Password, client_id, client_secret)
oauthData, err = oauth_login(endpoint, &o, extaccount.Username, extaccount.Password)
if err != nil {
log.Print("Unable to login: ", err)
return
@ -109,7 +110,7 @@ func PollMastodonPleroma(endpoint string, http_client http.Client) {
}
m.LastRun = time.Now().Format(time.RFC3339)
resp, err := http_client.Do(req)
resp, err := o.client.Do(req)
if err != nil {
m.Status = CLIENT_ISSUE
ri_mutex.Lock()

12
stream.go
View File

@ -9,11 +9,11 @@ import (
"time"
)
func StreamMastodon(endpoint string) {
func StreamMastodon(endpoint string, o *RunningInstance) {
http_client := http.Client{}
var client_id string
var client_secret string
//var client_id string
//var client_secret string
var oauthData OAuth
var err error
@ -26,14 +26,14 @@ func StreamMastodon(endpoint string) {
for _, extaccount := range settings.Externalaccounts {
if extaccount.Endpoint == endpoint {
get_client(endpoint, &http_client)
get_client(endpoint, o)
client_id, client_secret, err = get_client(endpoint, &http_client)
err = get_client(endpoint, o)
if err != nil {
log.Fatal("Unable to register client: ", err)
}
oauthData, err = oauth_login(endpoint, extaccount.Username, extaccount.Password, client_id, client_secret)
oauthData, err = oauth_login(endpoint, o, extaccount.Username, extaccount.Password)
if err != nil {
log.Print("Unable to login: ", err)
return