intermediate commit, node type determination, working on time format

lots of crashing

poll/headers.go

@@ -4,8 +4,11 @@ const (
 	NEW_INSTANCE =			0
 	RUNNING =			200
 	UNAUTHORIZED =			401
+	FORBIDDEN =			403
+	NOT_FOUND =			404
 	UNPROCESSABLE_ENTITY =		422
 	TOOMANYREQUESTS =		429
+	INTERNAL_ERROR =		500
 	CLIENT_ISSUE =			600
 	ONION_PROTOCOL =		601
 	BAD_RESPONSE =			602

poll/instance.go

@@ -20,7 +20,7 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
 
 	min_id := ""
 
-	http_client := http.Client{Timeout: 5 * time.Second}
+	http_client := http.Client{}
 
 	for {
 		ri_mutex.Lock()
@@ -35,10 +35,12 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
 			m.Status = CLIENT_ISSUE
 			runninginstances[endpoint] = m
 			ri_mutex.Unlock()
+			log.Fatal("Failure here", err.Error())
 			return
 		}
 
 		if resp.StatusCode == UNAUTHORIZED || resp.StatusCode == UNPROCESSABLE_ENTITY {
+			// Apparently you just need to do this to throw away the body
 			_, _ = ioutil.ReadAll(resp.Body)
 			resp.Body.Close() // Release as soon as done
 			ri_mutex.Lock()
@@ -56,15 +58,53 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
 			ri_mutex.Unlock()
 			time.Sleep(time.Second * 30)
 			continue
+		} else if resp.StatusCode == FORBIDDEN {
+			// Apparently you just need to do this to throw away the body
+			_, _ = ioutil.ReadAll(resp.Body)
+			resp.Body.Close() // Release as soon as done
+			ri_mutex.Lock()
+			m.Status = FORBIDDEN
+			runninginstances[endpoint] = m
+			ri_mutex.Unlock()
+			return
+		} else if resp.StatusCode == INTERNAL_ERROR {
+			// Apparently you just need to do this to throw away the body
+			_, _ = ioutil.ReadAll(resp.Body)
+			resp.Body.Close() // Release as soon as done
+			ri_mutex.Lock()
+			m.Status = FORBIDDEN
+			runninginstances[endpoint] = m
+			ri_mutex.Unlock()
+			time.Sleep(time.Second * 3600)
+			continue
+		} else if resp.StatusCode == NOT_FOUND { // 404
+			// Apparently you just need to do this to throw away the body
+			_, _ = ioutil.ReadAll(resp.Body)
+			resp.Body.Close() // Release as soon as done
+			ri_mutex.Lock()
+			m.Status = FORBIDDEN
+			runninginstances[endpoint] = m
+			ri_mutex.Unlock()
+			time.Sleep(time.Second * 3600)
+			continue
 		}
+
 		err = json.NewDecoder(resp.Body).Decode(&newposts)
 		if err != nil {
-			fmt.Println("Error Message 1:", resp.StatusCode, err, endpoint, resp.Status)
+			fmt.Println("-----------------")
+			fmt.Println("Error Message 1:", resp.StatusCode, err, err.Error, endpoint, resp.Status)
+			fmt.Println("-----------------")
+			q, err := ioutil.ReadAll(resp.Body)
+			fmt.Println(string(q))
+			fmt.Println(q)
+			fmt.Println(err)
+			resp.Body.Close() // Release as soon as done
 			ri_mutex.Lock()
 			m.Status = BAD_RESPONSE
 			runninginstances[endpoint] = m
 			ri_mutex.Unlock()
-			return
+			log.Fatal("Exiting")
+			continue
 		}
 		resp.Body.Close() // Release as soon as done
 
@@ -96,10 +136,30 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
 			newpost.normalized = html.UnescapeString(strings.ToLower(p.Sanitize(newpost.Content)))
 
 			// Validate time
-			_, err := time.Parse("2006-01-02T15:04:05.999Z", newpost.Created_at)
+			//t, err := time.Parse("2006-01-02T15:04:05.999Z", newpost.Created_at)
+			t, err := time.Parse(time.RFC3339, newpost.Created_at)
 			if err != nil {
-				fmt.Println("---------> FIXED TIME", newpost.Created_at)
+				log.Print("Time was: " + newpost.Created_at)
-				newpost.Created_at = time.Now().Format("2006.01.02-15:04:05")
+				newpost.Created_at = time.Now().Format(time.RFC3339)
+				log.Print("Set to  : " + newpost.Created_at)
+			}
+			if t.Unix() < 0 {
+				log.Print("Time was: " + newpost.Created_at)
+				newpost.Created_at = time.Now().Format(time.RFC3339)
+				log.Print("Set to  : " + newpost.Created_at)
+			}
+
+			//t, err = time.Parse("2006-01-02T15:04:05.999Z", newpost.Account.Created_at)
+			t, err = time.Parse(time.RFC3339, newpost.Created_at)
+			if err != nil {
+				log.Print("Time was: " + newpost.Account.Created_at)
+				newpost.Account.Created_at = time.Now().Format(time.RFC3339)
+				log.Print("Set to  : " + newpost.Account.Created_at)
+			}
+			if t.Unix() < 0 {
+				log.Print("Time was: " + newpost.Account.Created_at)
+				newpost.Account.Created_at = time.Now().Format(time.RFC3339)
+				log.Print("Set to  : " + newpost.Account.Created_at)
 			}
 
 			reportPostChan <- newpost
@@ -125,51 +185,54 @@ func PollMastodonPleroma(endpoint string, reportPostChan chan ReportPost) {
 
 // Change this to return a proper "err"
 func GetNodeInfo(endpoint string) (NodeInfo) {
-	var nodeinfo NodeInfo
+	/* Checking order
-
+	 * Mastodon/Pleroma
-	/* Not everyone implements this */
+	 * Um..nothing else yet
-	api_nodeinfo := "https://" + endpoint + "/nodeinfo/2.0.json"
-	http_client := http.Client{Timeout: 10 * time.Second}
-	resp_node, err := http_client.Get(api_nodeinfo)
-	if err != nil {
-		fmt.Println("Make a legit error here", err)
-		return NodeInfo{}
-	}
-	defer resp_node.Body.Close()
-
-	/*
-		Some Pleroma and Mastodon instances hide their /nodeinfo/2.0.json
-		So this attempts to check the index URL for references to Mastodon
-		or Pleroma.
 	*/
-	if resp_node.StatusCode == 200 {
+	pleromastodon_nodeinfo_url := "https://" + endpoint + "/nodeinfo/2.0.json"
-		err = json.NewDecoder(resp_node.Body).Decode(&nodeinfo)
+	//http_client := http.Client{Timeout: 10 * time.Second}
+	http_client := http.Client{}
+	pleromastodon_api_resp, err := http_client.Get(pleromastodon_nodeinfo_url)
 	if err != nil {
-			fmt.Println("Error Message 2:", resp_node.StatusCode, err, endpoint, resp_node.Status, api_nodeinfo)
 		return NodeInfo{}
+	} else {
+		defer pleromastodon_api_resp.Body.Close()
 	}
+
+	if pleromastodon_api_resp.StatusCode == 200 {
+		var nodeinfo NodeInfo
+		err = json.NewDecoder(pleromastodon_api_resp.Body).Decode(&nodeinfo)
+		if err != nil {
+			log.Println("Was not unmarshalled, continuing...")
+		}
+		return nodeinfo
+	}
+
+	// Check the front page
+	index_url := "https://" + endpoint + "/"
+	resp_index, err := http_client.Get(index_url)
+	if err != nil {
+		log.Fatal("Error Message 2b:", resp_index.StatusCode, err, endpoint, resp_index.Status)
 	}
-	if resp_node.StatusCode == 404 {
-		indexurl := "https://" + endpoint + "/"
-		resp_index, err := http_client.Get(indexurl)
 	defer resp_index.Body.Close()
-		if err != nil {
-			fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo)
-			return NodeInfo{}
-		}
 	indexbin, err := ioutil.ReadAll(resp_index.Body)
 	if err != nil {
-			fmt.Println("Error Message 2:", resp_index.StatusCode, err, endpoint, resp_index.Status, api_nodeinfo)
+		log.Fatal("Error Message 2c:", resp_index.StatusCode, err, endpoint, resp_index.Status)
-			return NodeInfo{}
 	}
 	indexstr := string(indexbin)
-		if strings.Contains(indexstr, "Pleroma") {
+	nodeinfo := NodeInfo{}
+	if strings.Contains(indexstr, "Pleroma") || strings.Contains(indexstr, "Soapbox") {
+		log.Print("Manual view: Pleroma" + endpoint)
 		nodeinfo.Software.Name = "pleroma"
+		nodeinfo.Software.Version = "guess"
 	} else if strings.Contains(indexstr, "Mastodon") {
+		log.Print("Manual view: Mastodon" + endpoint)
 		nodeinfo.Software.Name = "mastodon"
-		} else {
+		nodeinfo.Software.Version = "guess"
-			return NodeInfo{}
+	} else if strings.Contains(indexstr, "Gab") {
-		}
+		log.Print("Manual view: Gab" + endpoint)
+		nodeinfo.Software.Name = "gab"
+		nodeinfo.Software.Version = "guess"
 	}
 
 	return nodeinfo
@@ -180,7 +243,7 @@ func StartInstance(endpoint string, reportPostChan chan ReportPost) {
 	if nodeinfo.Software.Name == "" {
 		var m = runninginstances[endpoint]
 		m.Software = ""
-		m.LastRun = time.Now().Format("2006.01.02-15:04:05")
+		m.LastRun = time.Now().Format(time.RFC3339)
 		m.Status = UNSUPPORTED_INSTANCE
 		ri_mutex.Lock()
 		runninginstances[endpoint] = m