farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

intermediate commit, node type determination, working on time format

Browse Source 
lots of crashing
This commit is contained in:
farhan 2020-11-25 18:24:56 -05:00
parent a81c61d5dc
commit 53279ff649
2 changed files with 114 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
poll/headers.go
View File

@ -4,8 +4,11 @@ const (
NEW_INSTANCE = 0
RUNNING = 200
UNAUTHORIZED = 401
FORBIDDEN = 403
NOT_FOUND = 404
UNPROCESSABLE_ENTITY = 422
TOOMANYREQUESTS = 429
INTERNAL_ERROR = 500
CLIENT_ISSUE = 600
ONION_PROTOCOL = 601
BAD_RESPONSE = 602

159
poll/instance.go
View File

@ -20,7 +20,7 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
min_id := ""
http_client := http.Client{Timeout: 5 * time.Second}
http_client := http.Client{}
for {
ri_mutex.Lock()
@ -35,10 +35,12 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
m.Status = CLIENT_ISSUE
runninginstances[endpoint] = m
ri_mutex.Unlock()
log.Fatal("Failure here", err.Error())
return
}
if resp.StatusCode == UNAUTHORIZED || resp.StatusCode == UNPROCESSABLE_ENTITY {
// Apparently you just need to do this to throw away the body
_, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close() // Release as soon as done
ri_mutex.Lock()
@ -56,15 +58,53 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
ri_mutex.Unlock()
time.Sleep(time.Second * 30)
continue
} else if resp.StatusCode == FORBIDDEN {
// Apparently you just need to do this to throw away the body
_, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close() // Release as soon as done
ri_mutex.Lock()
m.Status = FORBIDDEN
runninginstances[endpoint] = m
ri_mutex.Unlock()
return
} else if resp.StatusCode == INTERNAL_ERROR {
// Apparently you just need to do this to throw away the body
_, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close() // Release as soon as done
ri_mutex.Lock()
m.Status = FORBIDDEN
runninginstances[endpoint] = m
ri_mutex.Unlock()
time.Sleep(time.Second * 3600)
continue
} else if resp.StatusCode == NOT_FOUND { // 404
// Apparently you just need to do this to throw away the body
_, _ = ioutil.ReadAll(resp.Body)
resp.Body.Close() // Release as soon as done
ri_mutex.Lock()
m.Status = FORBIDDEN
runninginstances[endpoint] = m
ri_mutex.Unlock()
time.Sleep(time.Second * 3600)
continue
}
err = json.NewDecoder(resp.Body).Decode(&newposts)
if err != nil {
fmt.Println("Error Message 1:", resp.StatusCode, err, endpoint, resp.Status)
fmt.Println("-----------------")
fmt.Println("Error Message 1:", resp.StatusCode, err, err.Error, endpoint, resp.Status)
fmt.Println("-----------------")
q, err := ioutil.ReadAll(resp.Body)
fmt.Println(string(q))
fmt.Println(q)
fmt.Println(err)
resp.Body.Close() // Release as soon as done
ri_mutex.Lock()
m.Status = BAD_RESPONSE
runninginstances[endpoint] = m
ri_mutex.Unlock()
return
log.Fatal("Exiting")
continue
}
resp.Body.Close() // Release as soon as done
@ -96,10 +136,30 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content)))
// Validate time
_, err := time.Parse("2006-01-02T15:04:05.999Z", newpost.Created_at)
//t, err := time.Parse("2006-01-02T15:04:05.999Z", newpost.Created_at)
t, err := time.Parse(time.RFC3339, newpost.Created_at)
if err != nil {
fmt.Println("---------> FIXED TIME", newpost.Created_at)
newpost.Created_at = time.Now().Format("2006.01.02-15:04:05")
log.Print("Time was: " + newpost.Created_at)
newpost.Created_at = time.Now().Format(time.RFC3339)
log.Print("Set to : " + newpost.Created_at)
}
if t.Unix() < 0 {
log.Print("Time was: " + newpost.Created_at)
newpost.Created_at = time.Now().Format(time.RFC3339)
log.Print("Set to : " + newpost.Created_at)
}
//t, err = time.Parse("2006-01-02T15:04:05.999Z", newpost.Account.Created_at)
t, err = time.Parse(time.RFC3339, newpost.Created_at)
if err != nil {
log.Print("Time was: " + newpost.Account.Created_at)
newpost.Account.Created_at = time.Now().Format(time.RFC3339)
log.Print("Set to : " + newpost.Account.Created_at)
}
if t.Unix() < 0 {
log.Print("Time was: " + newpost.Account.Created_at)
newpost.Account.Created_at = time.Now().Format(time.RFC3339)
log.Print("Set to : " + newpost.Account.Created_at)
}
reportPostChan <- newpost
@ -125,51 +185,54 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
// Change this to return a proper "err"
func GetNodeInfo(endpoint string) (NodeInfo) {
var nodeinfo NodeInfo
/* Not everyone implements this */
api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json"
http_client := http.Client{Timeout: 10 * time.Second}
resp_node, err := http_client.Get(api_nodeinfo)
if err != nil {
fmt.Println("Make a legit error here", err)
return NodeInfo{}
}
defer resp_node.Body.Close()
/*
Some Pleroma and Mastodon instances hide their /nodeinfo/2.0.json
So this attempts to check the index URL for references to Mastodon
or Pleroma.
/* Checking order
* Mastodon/Pleroma
* Um..nothing else yet
*/
if resp_node.StatusCode == 200 {
err = json.NewDecoder(resp_node.Body).Decode(&nodeinfo)
if err != nil {
fmt.Println("Error Message 2:", resp_node.StatusCode, err, endpoint, resp_node.Status, api_nodeinfo)
return NodeInfo{}
}
pleromastodon_nodeinfo_url := "https://" + endpoint + "/nodeinfo/2.0.json"
//http_client := http.Client{Timeout: 10 * time.Second}
http_client := http.Client{}
pleromastodon_api_resp, err := http_client.Get(pleromastodon_nodeinfo_url)
if err != nil {
return NodeInfo{}
} else {
defer pleromastodon_api_resp.Body.Close()
}
if resp_node.StatusCode == 404 {
indexurl := "https://" + endpoint + "/"
resp_index, err := http_client.Get(indexurl)
defer resp_index.Body.Close()
if pleromastodon_api_resp.StatusCode == 200 {
var nodeinfo NodeInfo
err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo)
if err != nil {
fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo)
return NodeInfo{}
}
indexbin, err := ioutil.ReadAll(resp_index.Body)
if err != nil {
fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo)
return NodeInfo{}
}
indexstr := string(indexbin)
if strings.Contains(indexstr, "Pleroma") {
nodeinfo.Software.Name = "pleroma"
} else if strings.Contains(indexstr, "Mastodon") {
nodeinfo.Software.Name = "mastodon"
} else {
return NodeInfo{}
log.Println("Was not unmarshalled, continuing...")
}
return nodeinfo
}
// Check the front page
index_url := "https://" + endpoint + "/"
resp_index, err := http_client.Get(index_url)
if err != nil {
log.Fatal("Error Message 2b:", resp_index.StatusCode, err, endpoint, resp_index.Status)
}
defer resp_index.Body.Close()
indexbin, err := ioutil.ReadAll(resp_index.Body)
if err != nil {
log.Fatal("Error Message 2c:", resp_index.StatusCode, err, endpoint, resp_index.Status)
}
indexstr := string(indexbin)
nodeinfo := NodeInfo{}
if strings.Contains(indexstr, "Pleroma") || strings.Contains(indexstr, "Soapbox") {
log.Print("Manual view: Pleroma" + endpoint)
nodeinfo.Software.Name = "pleroma"
nodeinfo.Software.Version = "guess"
} else if strings.Contains(indexstr, "Mastodon") {
log.Print("Manual view: Mastodon" + endpoint)
nodeinfo.Software.Name = "mastodon"
nodeinfo.Software.Version = "guess"
} else if strings.Contains(indexstr, "Gab") {
log.Print("Manual view: Gab" + endpoint)
nodeinfo.Software.Name = "gab"
nodeinfo.Software.Version = "guess"
}
return nodeinfo
@ -180,7 +243,7 @@ func StartInstance(endpoint string, reportPostChan chan ReportPost) {
if nodeinfo.Software.Name == "" {
var m = runninginstances[endpoint]
m.Software = ""
m.LastRun = time.Now().Format("2006.01.02-15:04:05")
m.LastRun = time.Now().Format(time.RFC3339)
m.Status = UNSUPPORTED_INSTANCE
ri_mutex.Lock()
runninginstances[endpoint] = m