2020-11-10 21:53:46 -05:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"github.com/microcosm-cc/bluemonday"
|
|
|
|
"encoding/json"
|
|
|
|
"crypto/sha1"
|
|
|
|
"io/ioutil"
|
|
|
|
"net/http"
|
|
|
|
"strings"
|
|
|
|
"html"
|
|
|
|
"time"
|
|
|
|
"fmt"
|
|
|
|
)
|
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
/*
|
|
|
|
func DeferPollRun(instancereport InstanceReport, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) {
|
2020-11-10 21:53:46 -05:00
|
|
|
|
|
|
|
delay := 10
|
|
|
|
if instancereport.status == RUNNING && instancereport.numposts <= 10 {
|
|
|
|
delay = 10
|
|
|
|
} else if instancereport.status == RUNNING && instancereport.numposts > 10 {
|
|
|
|
delay = 15
|
|
|
|
} else if instancereport.status == 429 {
|
|
|
|
delay = 30
|
|
|
|
} else {
|
|
|
|
fmt.Println("error, status code is ------------->: ", instancereport.status)
|
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
time.Sleep(time.Second * time.Duration(delay))
|
|
|
|
|
2020-11-16 00:10:33 -05:00
|
|
|
StartInstancePoll(instancereport, reportPostChan, instanceReportChan)
|
2020-11-10 21:53:46 -05:00
|
|
|
}
|
2020-11-17 18:28:59 -05:00
|
|
|
*/
|
2020-11-10 21:53:46 -05:00
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
|
|
|
|
fmt.Println("Arrived at PollMastodonPleroma for " + endpoint)
|
|
|
|
// Make this a global variable
|
2020-11-10 21:53:46 -05:00
|
|
|
p := bluemonday.NewPolicy()
|
|
|
|
newposts := make([]ReportPost, 0)
|
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
min_id := ""
|
2020-11-10 21:53:46 -05:00
|
|
|
|
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
http_client := http.Client{Timeout: 5 * time.Second}
|
2020-11-10 21:53:46 -05:00
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
for {
|
2020-11-10 21:53:46 -05:00
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
api_timeline := "https://" + endpoint + "/api/v1/timelines/public?limit=40&min_id=" + min_id
|
|
|
|
fmt.Println(api_timeline)
|
|
|
|
numposts := 0
|
|
|
|
// newinstances := make([]string, 0)
|
2020-11-10 21:53:46 -05:00
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
resp, err := http_client.Get(api_timeline)
|
|
|
|
if err != nil {
|
|
|
|
ri_mutex.Lock()
|
|
|
|
var m = runninginstances[endpoint]
|
|
|
|
m.Status = CLIENT_ISSUE
|
|
|
|
runninginstances[endpoint] = m
|
|
|
|
ri_mutex.Unlock()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
|
|
if err != nil {
|
|
|
|
ri_mutex.Lock()
|
|
|
|
var m = runninginstances[endpoint]
|
|
|
|
m.Status = BAD_RESPONSE
|
|
|
|
runninginstances[endpoint] = m
|
|
|
|
ri_mutex.Unlock()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
err = json.Unmarshal(body, &newposts)
|
|
|
|
if err != nil {
|
|
|
|
ri_mutex.Lock()
|
|
|
|
var m = runninginstances[endpoint]
|
|
|
|
m.Status = BAD_RESPONSE
|
|
|
|
runninginstances[endpoint] = m
|
|
|
|
ri_mutex.Unlock()
|
|
|
|
return
|
2020-11-10 21:53:46 -05:00
|
|
|
}
|
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
resp.Body.Close()
|
2020-11-10 21:53:46 -05:00
|
|
|
|
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
for _, newpost := range newposts {
|
|
|
|
if newpost.Account.Acct == "" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
posthash := sha1.New()
|
2020-11-10 21:53:46 -05:00
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
at_sign := strings.Index(newpost.Account.Acct, "@")
|
|
|
|
|
|
|
|
if at_sign == -1 {
|
|
|
|
at_sign = len(newpost.Account.Acct)
|
|
|
|
newpost.Account.Acct += "@" + endpoint
|
|
|
|
}
|
|
|
|
|
|
|
|
// Calculate the post hash
|
|
|
|
fmt.Fprint(posthash, newpost.Url)
|
|
|
|
fmt.Fprint(posthash, newpost.normalized)
|
|
|
|
fmt.Fprint(posthash, newpost.Account.Acct)
|
|
|
|
fmt.Fprint(posthash, newpost.Account.Display_name)
|
|
|
|
newpost.posthash = posthash.Sum(nil)
|
|
|
|
|
|
|
|
newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content)))
|
|
|
|
|
|
|
|
reportPostChan <- newpost
|
|
|
|
|
|
|
|
// Check min_id
|
|
|
|
if newpost.Id > min_id {
|
|
|
|
min_id = newpost.Id
|
|
|
|
}
|
|
|
|
numposts = numposts + 1
|
|
|
|
|
|
|
|
newinstance := newpost.Account.Acct[at_sign+1:]
|
|
|
|
ri_mutex.Lock()
|
|
|
|
_, exists := runninginstances[newinstance]
|
|
|
|
if exists == false {
|
|
|
|
m := RunningInstance{}
|
|
|
|
runninginstances[newinstance] = m
|
|
|
|
go StartInstance(newinstance, reportPostChan)
|
|
|
|
}
|
|
|
|
|
|
|
|
ri_mutex.Unlock()
|
2020-11-10 21:53:46 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
time.Sleep(time.Second * 10)
|
2020-11-10 21:53:46 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// Change this to return a proper "err"
|
2020-11-17 18:28:59 -05:00
|
|
|
func GetNodeInfo(endpoint string) (NodeInfo) {
|
|
|
|
var nodeinfo NodeInfo
|
2020-11-10 21:53:46 -05:00
|
|
|
api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json"
|
|
|
|
http_client := http.Client{Timeout: 5 * time.Second}
|
|
|
|
resp, err := http_client.Get(api_nodeinfo)
|
|
|
|
if err != nil {
|
2020-11-17 18:28:59 -05:00
|
|
|
fmt.Println("Make a legit error here")
|
|
|
|
return NodeInfo{}
|
2020-11-10 21:53:46 -05:00
|
|
|
}
|
2020-11-16 00:10:33 -05:00
|
|
|
defer resp.Body.Close()
|
2020-11-10 21:53:46 -05:00
|
|
|
|
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
|
|
err = json.Unmarshal(body, &nodeinfo)
|
|
|
|
if err != nil {
|
2020-11-17 18:28:59 -05:00
|
|
|
fmt.Println("Make a legit error here")
|
|
|
|
fmt.Println("Unmarshal 2");
|
|
|
|
return NodeInfo{}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nodeinfo
|
|
|
|
}
|
|
|
|
|
|
|
|
func StartInstance(endpoint string, reportPostChan chan ReportPost) {
|
|
|
|
|
|
|
|
// This might not be necessary...
|
|
|
|
// ri_mutex.Lock()
|
|
|
|
// _, exists := runninginstances[endpoint]
|
|
|
|
// fmt.Println("The exists is", exists)
|
|
|
|
// ri_mutex.Unlock()
|
|
|
|
// if exists == true {
|
|
|
|
// return
|
|
|
|
// }
|
|
|
|
|
|
|
|
nodeinfo := GetNodeInfo(endpoint)
|
|
|
|
if nodeinfo.Software.Name == "" {
|
|
|
|
ri_mutex.Lock()
|
|
|
|
var m = runninginstances[endpoint]
|
|
|
|
m.Software = ""
|
|
|
|
runninginstances[endpoint] = m
|
|
|
|
ri_mutex.Unlock()
|
2020-11-10 21:53:46 -05:00
|
|
|
return
|
|
|
|
}
|
2020-11-17 18:28:59 -05:00
|
|
|
|
|
|
|
if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" {
|
|
|
|
go PollMastodonPleroma(endpoint, reportPostChan)
|
|
|
|
}
|
|
|
|
|
2020-11-10 21:53:46 -05:00
|
|
|
}
|
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
/*
|
|
|
|
func NewInstance(endpoint string, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) {
|
2020-11-10 21:53:46 -05:00
|
|
|
var nodeinfo NodeInfo
|
|
|
|
|
|
|
|
if endpoint == "" {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// No repeats
|
2020-11-17 18:28:59 -05:00
|
|
|
|
|
|
|
for _, runninginstance := range runninginstances {
|
2020-11-10 21:53:46 -05:00
|
|
|
if runninginstance.Endpoint == endpoint {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check node type
|
|
|
|
GetNodeInfo(endpoint, &nodeinfo)
|
|
|
|
if nodeinfo.Software.Name == "" {
|
|
|
|
go func() {
|
|
|
|
var q InstanceReport
|
|
|
|
q.endpoint = endpoint
|
|
|
|
q.status = BAD_NODEINFO
|
|
|
|
instanceReportChan <- q
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
newinstance := RunningInstance{endpoint, "", "", NEW_INSTANCE, "Queued"}
|
2020-11-17 18:28:59 -05:00
|
|
|
runninginstances = append(runninginstances, newinstance)
|
2020-11-10 21:53:46 -05:00
|
|
|
|
|
|
|
if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" {
|
|
|
|
var newinstancereport InstanceReport
|
|
|
|
newinstancereport.endpoint = endpoint
|
|
|
|
newinstancereport.status = 0
|
|
|
|
newinstancereport.min_id = ""
|
|
|
|
newinstancereport.numposts = 0
|
|
|
|
go StartInstancePoll(newinstancereport, reportPostChan, instanceReportChan)
|
|
|
|
}
|
|
|
|
}
|
2020-11-17 18:28:59 -05:00
|
|
|
*/
|
2020-11-10 21:53:46 -05:00
|
|
|
|
2020-11-17 18:28:59 -05:00
|
|
|
/*
|
|
|
|
func SuspendInstance(suspendinstance InstanceReport) {
|
|
|
|
for i, runninginstance := range runninginstances {
|
2020-11-10 21:53:46 -05:00
|
|
|
if runninginstance.Endpoint == suspendinstance.endpoint {
|
2020-11-17 18:28:59 -05:00
|
|
|
(runninginstances)[i].Status = suspendinstance.status
|
|
|
|
(runninginstances)[i].LastRun = time.Now().Format("2006.01.02-15:04:05")
|
2020-11-10 21:53:46 -05:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2020-11-17 18:28:59 -05:00
|
|
|
*/
|