trending-linux-packages/trending-pkgs.go

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"sort"
	"strings"

	"github.com/montanaflynn/stats"
)

const API string = "https://pkgstats.archlinux.de/api/"

type SomeStruct struct {
	Name   string
	Zscore float64
}

type PackagePopularity struct {
	Name       string  `json:"name"`
	Samples    int     `json:"samples"`
	Count      int     `json:"count"`
	Popularity float64 `json:"popularity"`
	StartMonth int     `json:"startMonth"`
	EndMonth   int     `json:"endMonth"`
}

type Response struct {
	Total               int                 `json:"total"`
	Count               int                 `json:"count"`
	Limit               int                 `json:"limit"`
	Offset              int                 `json:"offset"`
	Query               interface{}         `json:"query"`
	PackagePopularities []PackagePopularity `json:"packagePopularities"`
}

func main() {
	//allPackages := get_package_names()

	// Read package names from packages.txt file
	packageNames, err := readLinesFromFile("packages.txt")
	if err != nil {
		log.Fatal(err)
	}

	// Get non-dependency packages
	nonDependencyPackages := getNonDependencyPackages(packageNames)

	// Write package names to files
	//writeToFile(allPackages, "packages.txt")
  err = writeToFile(nonDependencyPackages, "non_dependency_packages.txt")
	if err != nil {
		log.Fatal(err)
	}

	// Get and print trending packages
	trending := get_trending_packages(packageNames)
	for _, _struct := range trending {
		fmt.Println(_struct.Name, _struct.Zscore)
	}
}

func get_trending_packages(pkgs []string) []SomeStruct {
	var trending []SomeStruct
	f, _ := os.Create("popularities.txt")
	defer f.Close()
	for _, pkg := range pkgs {
		popularities := get_package_popularities(pkg)
		fmt.Fprintln(f, pkg, popularities)
		trending = append(trending, SomeStruct{pkg, get_zscore(popularities)})
	}
	sort.Slice(trending, func(i, j int) bool { return trending[i].Zscore > trending[j].Zscore })
	return trending
}

func get_zscore(popularities []float64) float64 {
	mean, _ := stats.Mean(popularities)
	std, _ := stats.StandardDeviation(popularities)
	return (popularities[len(popularities)-1] - mean) / std
}

func get_total_packages() int {
	var response Response
	response = requestJSON("packages?limit=1")
	return response.Total
}

func get_package_names() []string {
	var packages []string
	var total = get_total_packages()
	var offset, limit int = 0, 100

	for offset < total {
		relURL := "packages?limit=" + fmt.Sprint(limit) + "&offset=" + fmt.Sprint(offset)
		var response Response
		response = requestJSON(relURL)
		for _, pkg := range response.PackagePopularities {
			packages = append(packages, pkg.Name)
		}
		offset += limit
	}
	return packages
}

func get_package_popularities(pkg string) []float64 {
	relURL := "packages/" + pkg + "/series?startMonth=201009"
	var response Response
	response = requestJSON(relURL)
	var popularities []float64
	for _, pkg := range response.PackagePopularities {
		popularities = append(popularities, pkg.Popularity)
	}
	return popularities
}

func requestJSON(relURL string) Response {
	resp, err := http.Get(API + relURL)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	var response Response
	err = json.Unmarshal(body, &response)
	if err != nil {
		log.Fatal(err)
	}
	return response
}

func getNonDependencyPackages(packages []string) []string {
	var nonDependencyPackages []string

	for _, pkg := range packages {
		cmd := exec.Command("pacman", "-Sii", pkg)
		output, err := cmd.Output()

		if err != nil {
			fmt.Println("Error while checking package", pkg, ":", err)
			continue
		}

		outputStr := string(output)
		if strings.Contains(outputStr, "Required By     : None") {
			nonDependencyPackages = append(nonDependencyPackages, pkg)
		}
	}

	return nonDependencyPackages
}

func writeToFile(data []string, fileName string) error {
    file, err := os.Create(fileName)
    if err != nil {
        return err
    }
    defer file.Close()

    for _, item := range data {
        _, err = fmt.Fprintln(file, item)
        if err != nil {
            return err
        }
    }

    return nil
}

func readLinesFromFile(filename string) ([]string, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var lines []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		lines = append(lines, line)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return lines, nil
}