feat: first version of API gathering data from MongoDB

Makefile

@@ -8,9 +8,17 @@ GOFUMPT=gofumpt -l -w
 #thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
 
 build: ## Builds the binary
-	@echo "Building binary"
+	@echo "Building import binary"
 	go build -o bin/importer cmd/importer/main.go
 
+build-storelinks: ## Builds the binary
+	@echo "Building storelinks binary"
+	go build -o bin/storelinks cmd/storelinks/main.go
+
+build-linksapi: ## Builds the binary
+	@echo "Building linksapi binary"
+	go build -o bin/linksapi cmd/linksapi/main.go
+
 lint: $(GOLANGCI) ## Runs golangci-lint with predefined configuration
 	@echo "Applying linter"
 	golangci-lint version

cmd/linksapi/main.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/kris-dev-hub/globallinks/pkg/linkdb"
+)
+
+func main() {
+	if len(os.Args) < 4 {
+		fmt.Println("Require database configuration : ./linksapi localhost 27017 linkdb")
+		os.Exit(1)
+	}
+
+	host := os.Args[1]
+	port := os.Args[2]
+	dbname := os.Args[3]
+
+	linkdb.InitServer(host, port, dbname)
+}

pkg/linkdb/controller.go

@@ -0,0 +1,150 @@
+package linkdb
+
+import (
+	"context"
+	"log"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func (app *App) ControllerGetDomainLinks(apiRequest APIRequest) ([]LinkOut, error) {
+	var links []LinkRow
+	var outLinks []LinkOut
+	var limit int64 = 100
+	var page int64 = 1
+
+	domain := *apiRequest.Domain
+	if apiRequest.Limit != nil && *apiRequest.Limit > 0 && *apiRequest.Limit <= 1000 {
+		limit = *apiRequest.Limit
+	}
+	if apiRequest.Page != nil && *apiRequest.Page > 0 {
+		page = *apiRequest.Page
+	}
+
+	// Get the collection
+	collection := app.DB.Database(app.Dbname).Collection("links")
+
+	// Create a filter for the query
+	filter := bson.M{"linkdomain": domain}
+	sort := bson.D{
+		{Key: "linkdomain", Value: 1},
+		{Key: "linkpath", Value: 1},
+		{Key: "linkrawquery", Value: 1},
+		{Key: "pagehost", Value: 1},
+		{Key: "pagepath", Value: 1},
+		{Key: "pagerawquery", Value: 1},
+		{Key: "datefrom", Value: 1},
+		{Key: "dateto", Value: 1},
+	}
+
+	findOptions := options.Find().SetSort(sort).SetLimit(limit).SetSkip((page - 1) * limit)
+
+	cursor, err := collection.Find(context.TODO(), filter, findOptions)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer cursor.Close(context.TODO())
+
+	// Iterate through the cursor
+	for cursor.Next(context.TODO()) {
+		var link LinkRow
+		if err := cursor.Decode(&link); err != nil {
+			return nil, err
+		}
+		links = append(links, link)
+	}
+
+	if err := cursor.Err(); err != nil {
+		return nil, err
+	}
+
+	outLinks = cleanDomainLinks(&links)
+
+	return outLinks, nil
+}
+
+func cleanDomainLinks(links *[]LinkRow) []LinkOut {
+	lastLink := LinkOut{}
+	curLink := LinkOut{}
+	outLinks := make([]LinkOut, 0, len(*links))
+	for _, link := range *links {
+		curLink = LinkOut{
+			LinkUrl:  showLinkScheme(link.LinkScheme) + "://" + showSubDomain(link.LinkSubDomain) + link.LinkDomain + showLinkPath(link.LinkPath) + showSubQuery(link.LinkRawQuery),
+			PageUrl:  showLinkScheme(link.PageScheme) + "://" + link.PageHost + showLinkPath(link.PagePath) + showSubQuery(link.PageRawQuery),
+			LinkText: link.LinkText,
+			NoFollow: link.NoFollow,
+			NoIndex:  link.NoIndex,
+			DateFrom: link.DateFrom,
+			DateTo:   link.DateTo,
+			IP:       []string{link.IP},
+			Qty:      link.Qty,
+		}
+
+		if lastLink.LinkUrl != curLink.LinkUrl || lastLink.PageUrl != curLink.PageUrl || lastLink.LinkText != curLink.LinkText || lastLink.NoFollow != curLink.NoFollow {
+			if lastLink.LinkUrl != "" {
+				outLinks = append(outLinks, lastLink)
+			}
+			lastLink = curLink
+			continue
+		}
+
+		if lastLink.DateFrom < curLink.DateFrom {
+			lastLink.DateFrom = curLink.DateFrom
+		}
+
+		if lastLink.DateTo > curLink.DateTo {
+			lastLink.DateTo = curLink.DateTo
+		}
+
+		addIPsToLink(&lastLink, &curLink)
+
+		lastLink.Qty += curLink.Qty
+
+	}
+
+	return outLinks
+}
+
+func showLinkScheme(scheme string) string {
+	if scheme == "1" {
+		return "http"
+	}
+	return "https"
+}
+
+func showSubDomain(subDomain string) string {
+	if subDomain == "" {
+		return ""
+	}
+	return subDomain + "."
+}
+
+func showSubQuery(rawQuery string) string {
+	if rawQuery == "" {
+		return ""
+	}
+	return "?" + rawQuery
+}
+
+func showLinkPath(linkPath string) string {
+	if linkPath == "" {
+		return "/"
+	}
+	return linkPath
+}
+
+func addIPsToLink(lastLink *LinkOut, curLink *LinkOut) {
+	alreadyExists := false
+	for _, ip := range lastLink.IP {
+		if ip == curLink.IP[0] {
+			alreadyExists = true
+			break
+		}
+	}
+
+	// If it's not already in the slice, append it
+	if !alreadyExists {
+		lastLink.IP = append(lastLink.IP, curLink.IP[0])
+	}
+}

pkg/linkdb/error.go

@@ -0,0 +1,13 @@
+package linkdb
+
+import "encoding/json"
+
+// GenerateError - generate error response
+func GenerateError(errorCode string, errorFunction string, errorInfo string) []byte {
+	errorData := new(ApiError)
+	errorData.ErrorCode = errorCode
+	errorData.Function = errorFunction
+	errorData.Error = errorInfo
+	jsonError, _ := json.Marshal(errorData)
+	return jsonError
+}

pkg/linkdb/handler.go

@@ -0,0 +1,56 @@
+package linkdb
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+
+	"github.com/kris-dev-hub/globallinks/pkg/commoncrawl"
+)
+
+// SendResponse - send http response
+func SendResponse(w http.ResponseWriter, status int, data []byte) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	if _, err := w.Write(data); err != nil {
+		log.Printf("error writing response: %v", err)
+	}
+}
+
+// HandlerGetDomainLinks - get domain links
+func (app *App) HandlerGetDomainLinks(w http.ResponseWriter, r *http.Request) {
+	var apiRequest APIRequest
+	decoder := json.NewDecoder(r.Body)
+	defer r.Body.Close()
+	err := decoder.Decode(&apiRequest)
+	if err != nil {
+		errorMsg := fmt.Sprintf("Error parsing request: %s", err)
+		SendResponse(w, http.StatusBadRequest, GenerateError("ErrorParsing", "HandlerGetDomainLinks", errorMsg))
+		return
+	}
+
+	if apiRequest.Domain == nil || *apiRequest.Domain == "" {
+		SendResponse(w, http.StatusBadRequest, GenerateError("ErrorNoDomain", "HandlerGetDomainLinks", "Domain is required"))
+		return
+	}
+
+	if !commoncrawl.IsValidDomain(*apiRequest.Domain) {
+		SendResponse(w, http.StatusBadRequest, GenerateError("ErrorInvalidDomain", "HandlerGetDomainLinks", "Invalid domain"))
+		return
+	}
+
+	links, err := app.ControllerGetDomainLinks(apiRequest)
+	if err != nil {
+		SendResponse(w, http.StatusInternalServerError, GenerateError("ErrorFailedLinks", "HandlerGetDomainLinks", "Error getting links"))
+		return
+	}
+
+	response, err := json.Marshal(links)
+	if err != nil {
+		SendResponse(w, http.StatusInternalServerError, GenerateError("ErrorJson", "HandlerGetDomainLinks", "Error marshalling links"))
+		return
+	}
+
+	SendResponse(w, http.StatusOK, response)
+}

pkg/linkdb/linkdb.go

@@ -0,0 +1,61 @@
+package linkdb
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+type App struct {
+	DB     *mongo.Client
+	Dbname string
+}
+
+func InitServer(host string, port string, dbname string) {
+	db, err := InitDB("mongodb://" + host + ":" + port)
+	if err != nil {
+		log.Fatal(err)
+	}
+	app := &App{DB: db, Dbname: dbname}
+
+	router := InitRoutes(app)
+
+	handlerWithCORS := enableCORS(router)
+
+	// start http server
+	if err := http.ListenAndServe(":8010", handlerWithCORS); err != nil {
+		fmt.Println("Failed to set up server")
+		panic(err)
+	}
+}
+
+func InitDB(connectionString string) (*mongo.Client, error) {
+	client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(connectionString))
+	if err != nil {
+		return nil, err
+	}
+	// TODO: Additional setup like pinging the database to check the connection can be done here
+	return client, nil
+}
+
+func enableCORS(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Set CORS headers
+		w.Header().Set("Access-Control-Allow-Origin", "*") // allow any origin
+		w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
+		w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
+
+		// Check if the request is for CORS options
+		if r.Method == "OPTIONS" {
+			// Just return with the headers, don't pass the request along
+			return
+		}
+
+		// Pass down the request to the next handler (or middleware)
+		next.ServeHTTP(w, r)
+	})
+}

pkg/linkdb/models.go

@@ -0,0 +1,46 @@
+package linkdb
+
+// LinkRow - link row
+type LinkRow struct {
+	LinkDomain    string `json:"link_domain"`
+	LinkSubDomain string `json:"link_sub_domain"`
+	LinkPath      string `json:"link_path"`
+	LinkRawQuery  string `json:"link_raw_query"`
+	LinkScheme    string `json:"link_scheme"`
+	PageHost      string `json:"page_host"`
+	PagePath      string `json:"page_path"`
+	PageRawQuery  string `json:"page_raw_query"`
+	PageScheme    string `json:"page_scheme"`
+	LinkText      string `json:"link_text"`
+	NoFollow      int    `json:"no_follow"`
+	NoIndex       int    `json:"no_index"`
+	DateFrom      string `json:"date_from"`
+	DateTo        string `json:"date_to"`
+	IP            string `json:"ip"`
+	Qty           int    `json:"qty"`
+}
+
+// LinkOut - link output
+type LinkOut struct {
+	LinkUrl  string   `json:"link_url"`
+	PageUrl  string   `json:"page_url"`
+	LinkText string   `json:"link_text"`
+	NoFollow int      `json:"no_follow"`
+	NoIndex  int      `json:"no_index"`
+	DateFrom string   `json:"date_from"`
+	DateTo   string   `json:"date_to"`
+	IP       []string `json:"ip"`
+	Qty      int      `json:"qty"`
+}
+
+type APIRequest struct {
+	Domain *string `json:"domain,omitempty"`
+	Limit  *int64  `json:"limit,omitempty"`
+	Page   *int64  `json:"page,omitempty"`
+}
+
+type ApiError struct {
+	ErrorCode string `json:"errorCode"`
+	Function  string `json:"function"`
+	Error     string `json:"error"`
+}