farhan/fedilogue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed Poll channel, merged into InstanceReport channel

Browse Source 
Found a bug where the main routine was sending to a channel it also
read from. If full, this resulted in the main channel blocking
Added pperf, removed Peer "hunting" code
This commit is contained in:
farhan 2020-10-31 04:24:53 +00:00
parent 1d71e8217a
commit 4050e336c3
1 changed files with 106 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
poll/engine.go
View File

@ -3,6 +3,7 @@ package main
import ( import (
"github.com/microcosm-cc/bluemonday" "github.com/microcosm-cc/bluemonday"
"github.com/jackc/pgx/pgxpool" "github.com/jackc/pgx/pgxpool"
_ "net/http/pprof"
"encoding/binary" "encoding/binary"
"encoding/json" "encoding/json"
"crypto/sha1" "crypto/sha1"
@ -20,21 +21,16 @@ import (
) )
const ( const (
NEW_INSTANCE = 0
INSTANCE_ERROR = -1 INSTANCE_ERROR = -1
NEW_INSTANCE = 0
RUNNING = 200
TOOMANYREQUESTS = 429
CLIENT_ISSUE = 600 CLIENT_ISSUE = 600
ONION_PROTOCOL = 601 ONION_PROTOCOL = 601
UNMARSHAL_ERROR = 602 UNMARSHAL_ERROR = 602
NO_CONNECTION = 603 NO_CONNECTION = 603
) )
type PollMessage struct {
from string
status int
min_id string
numposts int
}
// Parsing Unmarshal JSON type // Parsing Unmarshal JSON type
type ReportPost struct { type ReportPost struct {
@ -60,10 +56,12 @@ type AccountType struct {
} }
// Used to report a new instance to main // Used to report a new instance to main
type ReportInstance struct { type InstanceReport struct {
from string
endpoint string endpoint string
status int status int
min_id string
numposts int
} }
// Instance's new min_id value // Instance's new min_id value
@ -94,7 +92,7 @@ type ResponseBack struct {
RunningInstances []RunningInstance `json:"RunningInstances"` RunningInstances []RunningInstance `json:"RunningInstances"`
} }
func handleClient(commandClient net.Conn, runninginstances *[]RunningInstance, reportInstanceChan chan ReportInstance) { func handleClient(commandClient net.Conn, runninginstances *[]RunningInstance, instanceReportChan chan InstanceReport) {
sizebyte := make([]byte, 4) sizebyte := make([]byte, 4)
var commandmap CommandMap var commandmap CommandMap
@ -120,16 +118,14 @@ func handleClient(commandClient net.Conn, runninginstances *[]RunningInstance, r
switch commandmap.Type { switch commandmap.Type {
case "status": case "status":
fmt.Println("Status")
responseback.Message = "Ok" responseback.Message = "Ok"
case "add": case "add":
fmt.Println("Add instance: " + commandmap.Endpoint) fmt.Println("Add instance: " + commandmap.Endpoint)
var q ReportInstance var q InstanceReport
q.from = ""
q.endpoint = commandmap.Endpoint q.endpoint = commandmap.Endpoint
q.status = NEW_INSTANCE q.status = NEW_INSTANCE
reportInstanceChan <- q instanceReportChan <- q
responseback.Message = "Added " + commandmap.Endpoint responseback.Message = "Added " + commandmap.Endpoint
case "suspend": case "suspend":
@ -144,8 +140,6 @@ func handleClient(commandClient net.Conn, runninginstances *[]RunningInstance, r
responseback.Type = "status" responseback.Type = "status"
responseback.RunningInstances = *runninginstances responseback.RunningInstances = *runninginstances
fmt.Println(responseback)
responsebytes, err := json.Marshal(responseback) responsebytes, err := json.Marshal(responseback)
if err != nil { if err != nil {
fmt.Println("Error: ", err) fmt.Println("Error: ", err)
@ -155,7 +149,6 @@ func handleClient(commandClient net.Conn, runninginstances *[]RunningInstance, r
n = len(responsebytes) n = len(responsebytes)
binary.LittleEndian.PutUint32(sizebyte, uint32(n)) binary.LittleEndian.PutUint32(sizebyte, uint32(n))
fmt.Println(sizebyte)
commandClient.Write(sizebyte) commandClient.Write(sizebyte)
responsebyte, err := json.Marshal(responseback) responsebyte, err := json.Marshal(responseback)
@ -165,7 +158,6 @@ func handleClient(commandClient net.Conn, runninginstances *[]RunningInstance, r
} }
commandClient.Write(responsebyte) commandClient.Write(responsebyte)
commandClient.Close() commandClient.Close()
} }
@ -189,40 +181,34 @@ func AppendIfMissing(hay []string, needle string) []string {
return append(hay, needle) return append(hay, needle)
} }
func StartInstancePoll(endpoint string, min_id string, reportPostChan chan ReportPost, pollMessageChan chan PollMessage, reportInstanceChan chan ReportInstance) { func StartInstancePoll(instancereport InstanceReport, reportPostChan chan ReportPost, instanceReportChan chan InstanceReport) {
p := bluemonday.NewPolicy() p := bluemonday.NewPolicy()
newposts := make([]ReportPost, 0) newposts := make([]ReportPost, 0)
// Only placing this here to later have the option of using an HTTP client via a SOCKS5 Tor proxy // Only placing this here to later have the option of using
if strings.Contains(endpoint, ".onion") == true { // an HTTP client via a SOCKS5 Tor proxy
reportInstanceChan <- ReportInstance{endpoint, endpoint, ONION_PROTOCOL} if strings.Contains(instancereport.endpoint, ".onion") == true {
instanceReportChan <- InstanceReport{instancereport.endpoint, ONION_PROTOCOL, "", 0}
return return
} }
api_timeline := "https://" + endpoint + "/api/v1/timelines/public?min_id=" + min_id api_timeline := "https://" + instancereport.endpoint + "/api/v1/timelines/public?min_id=" + instancereport.min_id
resp, err := http.Get(api_timeline) resp, err := http.Get(api_timeline)
if err != nil { if err != nil {
reportInstanceChan <- ReportInstance{endpoint, endpoint, CLIENT_ISSUE} instanceReportChan <- InstanceReport{instancereport.endpoint, CLIENT_ISSUE, "", 0}
return return
} }
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &newposts) err = json.Unmarshal(body, &newposts)
if err != nil { if err != nil {
fmt.Println("Unmarshal 3"); // instanceReportChan <- InstanceReport{instancereport.endpoint, resp.StatusCode, "", 0}
// Perhaps get rid of this if-condition? instanceReportChan <- InstanceReport{instancereport.endpoint, 999, "", 0}
if resp.StatusCode >= 400 && resp.StatusCode < 500 {
reportInstanceChan <- ReportInstance{endpoint, endpoint, resp.StatusCode}
} else if resp.StatusCode >= 500 && resp.StatusCode < 600 {
reportInstanceChan <- ReportInstance{endpoint, endpoint, resp.StatusCode}
} else {
reportInstanceChan <- ReportInstance{endpoint, endpoint, UNMARSHAL_ERROR}
}
//log.Fatal(err)
return return
} }
newinstances := make([]string, 0) newinstances := make([]string, 0)
min_id := ""
numposts := 0 numposts := 0
for _, newpost := range newposts { for _, newpost := range newposts {
posthash := sha1.New() posthash := sha1.New()
@ -231,7 +217,7 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
if at_sign == -1 { if at_sign == -1 {
at_sign = len(newpost.Account.Acct) at_sign = len(newpost.Account.Acct)
newpost.Account.Acct += "@" + endpoint newpost.Account.Acct += "@" + instancereport.endpoint
} }
// Calculate the post hash // Calculate the post hash
@ -252,28 +238,29 @@ func StartInstancePoll(endpoint string, min_id string, reportPostChan chan Repor
numposts = numposts + 1 numposts = numposts + 1
newinstance := newpost.Account.Acct[at_sign+1:] newinstance := newpost.Account.Acct[at_sign+1:]
newinstances = AppendIfMissing(newinstances, newinstance) newinstances = AppendIfMissing(newinstances, newinstance)
} }
for _, newinstance := range newinstances { for _, newinstance := range newinstances {
var q ReportInstance var q InstanceReport
q.from = endpoint
q.endpoint = newinstance q.endpoint = newinstance
q.status = NEW_INSTANCE q.status = NEW_INSTANCE
reportInstanceChan <- q q.min_id = ""
q.numposts = 0
instanceReportChan <- q
} }
pollMessageChan <- PollMessage{endpoint, resp.StatusCode, min_id, numposts} instanceReportChan <- InstanceReport{instancereport.endpoint, resp.StatusCode, min_id, numposts}
} }
func StartGetPeers(endpoint string, reportInstanceChan chan ReportInstance) { /*
func StartGetPeers(endpoint string, instanceReportChan chan InstanceReport) {
var newpeers []string var newpeers []string
api_peers := "https://" + endpoint + "/api/v1/instance/peers" api_peers := "https://" + endpoint + "/api/v1/instance/peers"
resp, err := http.Get(api_peers) resp, err := http.Get(api_peers)
if err != nil { if err != nil {
reportInstanceChan <- ReportInstance{endpoint, endpoint, NO_CONNECTION} instanceReportChan <- InstanceReport{endpoint, NO_CONNECTION, "", 0}
return return
// os.Exit(1) // os.Exit(1)
} }
@ -284,48 +271,42 @@ func StartGetPeers(endpoint string, reportInstanceChan chan ReportInstance) {
if err != nil { if err != nil {
fmt.Println("Unmarshal 1"); fmt.Println("Unmarshal 1");
log.Fatal(err) log.Fatal(err)
reportInstanceChan <- ReportInstance{endpoint, endpoint, UNMARSHAL_ERROR} instanceReportChan <- InstanceReport{endpoint, endpoint, UNMARSHAL_ERROR, "", 0}
return return
} }
for _, newpeer := range newpeers { for _, newpeer := range newpeers {
var q ReportInstance var q InstanceReport
q.from = endpoint
q.endpoint = newpeer q.endpoint = newpeer
q.status = NEW_INSTANCE q.status = NEW_INSTANCE
reportInstanceChan <- q instanceReportChan <- q
} }
} }
*/
func DeferPollRun(pollmessage PollMessage, runninginstances *[]RunningInstance, reportInstanceChan chan ReportInstance, reportPostChan chan ReportPost, pollMessageChan chan PollMessage) { func DeferPollRun(instancereport InstanceReport, runninginstances *[]RunningInstance, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) {
var min_id string
for _, runninginstance := range *runninginstances {
if runninginstance.Endpoint == pollmessage.from {
min_id = runninginstance.Min_id
break
}
}
delay := 10 delay := 10
if pollmessage.status == 200 && pollmessage.numposts <= 10 { if instancereport.status == 200 && instancereport.numposts <= 10 {
delay = 10 delay = 10
} else if pollmessage.status == 200 && pollmessage.numposts > 10 { } else if instancereport.status == 200 && instancereport.numposts > 10 {
delay = 15 delay = 15
} else if pollmessage.status == 429 { } else if instancereport.status == 429 {
delay = 30 delay = 30
} else { } else {
fmt.Println("error, status code is: ", pollmessage.status) fmt.Println("error, status code is ------------->: ", instancereport.status)
os.Exit(1)
} }
time.Sleep(time.Second * time.Duration(delay)) time.Sleep(time.Second * time.Duration(delay))
go StartInstancePoll(pollmessage.from, min_id, reportPostChan, pollMessageChan, reportInstanceChan) go StartInstancePoll(instancereport, reportPostChan, instanceReportChan)
} }
// Change this to return a proper "err" // Change this to return a proper "err"
func GetNodeInfo(endpoint string, nodeinfo *NodeInfo) { func GetNodeInfo(endpoint string, nodeinfo *NodeInfo) {
api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json" api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json"
resp, err := http.Get(api_nodeinfo) http_client := http.Client{Timeout: 5 * time.Second}
resp, err := http_client.Get(api_nodeinfo)
if err != nil { if err != nil {
return return
} }
@ -333,39 +314,49 @@ func GetNodeInfo(endpoint string, nodeinfo *NodeInfo) {
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(body, &nodeinfo) err = json.Unmarshal(body, &nodeinfo)
if err != nil { if err != nil {
fmt.Println("Unmarshal 2"); // fmt.Println("Unmarshal 2");
return return
} }
} }
func NewInstance(endpoint string, runninginstances *[]RunningInstance, reportInstanceChan chan ReportInstance, reportPostChan chan ReportPost, pollMessageChan chan PollMessage) { func NewInstance(endpoint string, runninginstances *[]RunningInstance, instanceReportChan chan InstanceReport, reportPostChan chan ReportPost) {
var nodeinfo NodeInfo var nodeinfo NodeInfo
// Check node type
GetNodeInfo(endpoint, &nodeinfo)
if nodeinfo.Software.Name == "" {
fmt.Println("was blank: ", nodeinfo.Software.Name)
var q ReportInstance
q.from = ""
q.endpoint = endpoint
q.status = INSTANCE_ERROR
reportInstanceChan <- q
}
if endpoint == "" { if endpoint == "" {
return return
} }
// No repeats
for _, runninginstance := range *runninginstances { for _, runninginstance := range *runninginstances {
if runninginstance.Endpoint == endpoint { if runninginstance.Endpoint == endpoint {
return return
} }
} }
// Check node type
GetNodeInfo(endpoint, &nodeinfo)
if nodeinfo.Software.Name == "" {
go func() {
var q InstanceReport
q.endpoint = endpoint
q.status = INSTANCE_ERROR
instanceReportChan <- q
return
}()
}
newinstance := RunningInstance{endpoint, "", "", NEW_INSTANCE} newinstance := RunningInstance{endpoint, "", "", NEW_INSTANCE}
*runninginstances = append(*runninginstances, newinstance) *runninginstances = append(*runninginstances, newinstance)
if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" { if nodeinfo.Software.Name == "pleroma" || nodeinfo.Software.Name == "mastodon" {
go StartInstancePoll(endpoint, "", reportPostChan, pollMessageChan, reportInstanceChan) var newinstancereport InstanceReport
newinstancereport.endpoint = endpoint
newinstancereport.status = 0
newinstancereport.min_id = ""
newinstancereport.numposts = 0
go StartInstancePoll(newinstancereport, reportPostChan, instanceReportChan)
// fmt.Println("Temporarily disabled Peer Hunting") // fmt.Println("Temporarily disabled Peer Hunting")
// go StartGetPeers(endpoint, reportInstanceChan) // go StartGetPeers(endpoint, instanceReportChan)
} }
} }
@ -415,25 +406,24 @@ func writePost(pool *pgxpool.Pool, reportpost ReportPost) {
} }
} }
func SuspendInstance(suspendinstance ReportInstance, runninginstances *[]RunningInstance) { func SuspendInstance(suspendinstance InstanceReport, runninginstances *[]RunningInstance) {
for i, runninginstance := range *runninginstances { for i, runninginstance := range *runninginstances {
if runninginstance.Endpoint == suspendinstance.endpoint { if runninginstance.Endpoint == suspendinstance.endpoint {
fmt.Println("Updated status to ", suspendinstance.status)
(*runninginstances)[i].Status = suspendinstance.status (*runninginstances)[i].Status = suspendinstance.status
return return
} }
} }
} }
func main() {
func engine() {
// Current instances // Current instances
runninginstances := make([]RunningInstance, 0) runninginstances := make([]RunningInstance, 0)
// Initial Setup // Initial Setup
reportPostChan := make(chan ReportPost, 100) reportPostChan := make(chan ReportPost, 2000)
reportInstanceChan := make(chan ReportInstance, 100) instanceReportChan := make(chan InstanceReport, 100)
pollMessageChan := make (chan PollMessage, 100)
// Setup Database // Setup Database
pool, err := pgxpool.Connect(context.Background(), "postgres://postgres@127.0.0.1/fedilogue") pool, err := pgxpool.Connect(context.Background(), "postgres://postgres@127.0.0.1/fedilogue")
@ -449,41 +439,46 @@ func main() {
} }
defer l.Close() defer l.Close()
for { commandClient := make(chan net.Conn)
commandClient := make(chan net.Conn)
go func(l net.Listener) {
for {
c, err := l.Accept()
if err != nil {
commandClient <- nil
return
}
commandClient <- c
}
}(l)
go func(l net.Listener) {
for { for {
select { c, err := l.Accept()
case c := <-commandClient: // New client connection if err != nil {
go handleClient(c, &runninginstances, reportInstanceChan) fmt.Println("Error on accept")
case p := <-pollMessageChan: // A poller ended commandClient <- nil
return
}
commandClient <- c
}
}(l)
for {
select {
case c := <-commandClient: // New client connection
go handleClient(c, &runninginstances, instanceReportChan)
case v := <-reportPostChan: // New Post
go writePost(pool, v)
case w := <-instanceReportChan: // Start or suspend instance
if w.status == NEW_INSTANCE {
NewInstance(w.endpoint, &runninginstances, instanceReportChan, reportPostChan)
} else if w.status == RUNNING || w.status == TOOMANYREQUESTS {
for i, runninginstance := range runninginstances { for i, runninginstance := range runninginstances {
if runninginstance.Endpoint == p.from { if runninginstance.Endpoint == w.endpoint {
runninginstances[i].Min_id = p.min_id runninginstances[i].Min_id = w.min_id
runninginstances[i].Status = w.status
} }
} }
go DeferPollRun(p, &runninginstances, reportInstanceChan, reportPostChan, pollMessageChan) go DeferPollRun(w, &runninginstances, instanceReportChan, reportPostChan)
case v := <-reportPostChan: // New Post } else {
go writePost(pool, v) SuspendInstance(w, &runninginstances)
case w := <-reportInstanceChan: // Start or suspend instance
if w.status == NEW_INSTANCE {
NewInstance(w.endpoint, &runninginstances, reportInstanceChan, reportPostChan, pollMessageChan)
} else {
fmt.Println("Error here failure ", w.status)
SuspendInstance(w, &runninginstances)
}
} }
} }
} }
} }
func main() {
go engine()
log.Println("serving on port 8080")
log.Fatal(http.ListenAndServe(":8080", nil))
}