3 Commits b95cebfed2 ... bea53a39a7

Author SHA1 Message Date
  Adam bea53a39a7 handle errors in HTTP api 2 months ago
  Adam a404e3f99c DRY when converting Version from/to version string 2 months ago
  Adam 6485862ff0 handle errors in initialising TRAFFIC 2 months ago

+ 6 - 0
api/structs/structs.go

@@ -9,6 +9,12 @@ const (
 	VEHICLE_DEPARTED
 )
 
+type ErrorResponse struct {
+	Success bool
+	Field   string
+	Message string
+}
+
 type Time struct {
 	Hour      uint
 	Minute    uint

+ 36 - 53
file/file.go

@@ -155,30 +155,22 @@ func ListVersions(path string, location *time.Location) ([]feeds.Version, error)
 	}
 	for _, txtFile := range trafficFiles {
 		if strings.Contains(txtFile.Name(), ".txz") {
-			validity := strings.Split(strings.Replace(txtFile.Name(), ".txz", "", 1), "_")
-			validFrom, err := time.ParseInLocation("20060102", validity[0], location)
+			versionString := strings.Replace(txtFile.Name(), ".txz", "", 1)
+			version, err := feeds.MakeVersion(versionString)
 			if err != nil {
 				return nil, err
 			}
-			validTill, err := time.ParseInLocation("20060102", validity[1], location)
-			if err != nil {
-				return nil, err
-			}
-			version := feeds.Version{
-				ValidFrom: validFrom,
-				ValidTill: validTill,
-			}
 			versions = append(versions, version)
 		}
 	}
 	return versions, nil
 }
 
-func UnpackTraffic(dataHome, feedName string) error {
+func unpackTraffic(dataHome, feedName string) error {
 	path := filepath.Join(dataHome, feedName)
 	files, err := ioutil.ReadDir(path)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	unpacked := map[string]bool{}
 	packed := map[string]bool{}
@@ -190,20 +182,18 @@ func UnpackTraffic(dataHome, feedName string) error {
 		}
 	}
 
-	for version, _ := range packed {
+	for version := range packed {
 		if !unpacked[version] {
 			trafficPath := filepath.Join(path, version) + ".txz"
 			versionPath := filepath.Join(path, version)
 			os.Mkdir(versionPath, 0644)
 			trafficFile, err := os.Open(trafficPath)
 			if err != nil {
-				log.Println(err)
 				return err
 			}
 			defer trafficFile.Close()
 			xzReader, err := xz.NewReader(trafficFile)
 			if err != nil {
-				log.Println(err)
 				return err
 			}
 			tarReader := tar.NewReader(xzReader)
@@ -213,18 +203,15 @@ func UnpackTraffic(dataHome, feedName string) error {
 					break
 				}
 				if err != nil {
-					log.Println(err)
 					return err
 				}
 				barePath := filepath.Join(versionPath, hdr.Name)
 				bareFile, err := os.Create(barePath)
 				if err != nil {
-					log.Println(err)
 					return err
 				}
 				defer bareFile.Close()
 				if _, err := io.Copy(bareFile, tarReader); err != nil {
-					log.Println(err)
 					return err
 				}
 			}
@@ -233,27 +220,22 @@ func UnpackTraffic(dataHome, feedName string) error {
 	return nil
 }
 
-func CleanOldVersions(dataHome, feedName string, location *time.Location) {
+func cleanOldVersions(dataHome, feedName string, location *time.Location) error {
 	now := time.Now().In(location)
 	path := filepath.Join(dataHome, feedName)
 	files, err := ioutil.ReadDir(path)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	versions := []feeds.Version{}
 	versionsMap := map[string]feeds.Version{}
 	for _, file := range files {
 		name := file.Name()
 		versionString := strings.Replace(name, ".txz", "", 1)
-		// todo func feeds.MakeVersion(string) Version
-		versionDates := strings.Split(versionString, "_")
-		validFrom, _ := time.Parse("20060102", versionDates[0])
-		validTill, _ := time.Parse("20060102", versionDates[1])
-		version := feeds.Version{
-			ValidFrom: validFrom,
-			ValidTill: validTill,
+		versionsMap[versionString], err = feeds.MakeVersion(versionString)
+		if err != nil {
+			return err
 		}
-		versionsMap[versionString] = version
 	}
 	for _, version := range versionsMap {
 		versions = append(versions, version)
@@ -261,24 +243,10 @@ func CleanOldVersions(dataHome, feedName string, location *time.Location) {
 	sort.Slice(versions, func(i, j int) bool {
 		return versions[i].ValidFrom.Before(versions[j].ValidFrom)
 	})
-	// todo func feeds.FindValidVersions([]Version, time.Time) []Versions
-	i := 0
-	for _, version := range versions {
-		if version.ValidFrom.Before(now) {
-			i++
-		} else {
-			break
-		}
-	}
-	if i > 0 {
-		i = i - 1
-	}
-	validVersions := versions[i:]
+	validVersions := feeds.FindValidVersions(versions, now)
 	validVersionsMap := map[string]bool{}
 	for _, version := range validVersions {
-		// todo func (v Version) String() string
-		versionString := version.ValidFrom.Format("20060102") + "_" + version.ValidTill.Format("20060102")
-		validVersionsMap[versionString] = true
+		validVersionsMap[version.String()] = true
 	}
 
 	for _, file := range files {
@@ -289,12 +257,13 @@ func CleanOldVersions(dataHome, feedName string, location *time.Location) {
 			os.RemoveAll(filePath)
 		}
 	}
+	return nil
 }
 
-func ReadIndexes(dataHome string, feedName string, codeIndex *traffic_structs.GlobalCodeIndex, versions []feeds.Version) {
+func readIndexes(dataHome string, feedName string, codeIndex *traffic_structs.GlobalCodeIndex, versions []feeds.Version) {
 	ix := *codeIndex
 	for _, v := range versions {
-		versionCode := traffic_structs.Validity(v.ValidFrom.Format("20060102") + "_" + v.ValidTill.Format("20060102"))
+		versionCode := traffic_structs.Validity(v.String())
 		index := traffic_structs.ReadCodeIndex(dataHome, feedName, versionCode)
 		if ix[feedName] == nil {
 			ix[feedName] = traffic_structs.FeedCodeIndex{}
@@ -304,10 +273,10 @@ func ReadIndexes(dataHome string, feedName string, codeIndex *traffic_structs.Gl
 	*codeIndex = ix
 }
 
-func ReadCalendar(dataHome, feedName string, schedules *traffic_structs.GlobalCalendar, versions []feeds.Version) {
+func readCalendar(dataHome, feedName string, schedules *traffic_structs.GlobalCalendar, versions []feeds.Version) {
 	s := *schedules
 	for _, v := range versions {
-		versionCode := traffic_structs.Validity(v.ValidFrom.Format("20060102") + "_" + v.ValidTill.Format("20060102"))
+		versionCode := traffic_structs.Validity(v.String())
 		schedule := traffic_structs.ReadCalendar(dataHome, feedName, versionCode)
 		if s[feedName] == nil {
 			s[feedName] = traffic_structs.FeedCalendar{}
@@ -326,13 +295,27 @@ func InitialiseTraffic(sigChan chan os.Signal, doneChan chan bool, cfg config.Co
 		}
 		allVersions := feeds.GlobalVersions{}
 		for _, feed := range cfg.EnabledFeeds {
-			UnpackTraffic(cfg.FeedsPath, feed.String())
-			CleanOldVersions(cfg.FeedsPath, feed.String(), feed.GetLocation())
 			feedHome := filepath.Join(cfg.FeedsPath, feed.String())
-			feedVersions, _ := ListVersions(feedHome, feed.GetLocation())
+
+			err := unpackTraffic(cfg.FeedsPath, feed.String())
+			if err != nil {
+				log.Printf("while unpacking TRAFFIC in feed %s: %v\n", feed, err)
+				continue
+			}
+			err = cleanOldVersions(cfg.FeedsPath, feed.String(), feed.GetLocation())
+			if err != nil {
+				log.Printf("while cleaning old TRAFFIC versions in feed %s: %v\n", feed, err)
+				continue
+			}
+			feedVersions, err := ListVersions(feedHome, feed.GetLocation())
+			if err != nil {
+				log.Printf("while listing TRAFFIC versions in feed %s: %v\n", feed, err)
+				continue
+			}
+
 			allVersions[feed.String()] = feedVersions
-			ReadIndexes(cfg.FeedsPath, feed.String(), codeIndex, feedVersions)
-			ReadCalendar(cfg.FeedsPath, feed.String(), calendar, feedVersions)
+			readIndexes(cfg.FeedsPath, feed.String(), codeIndex, feedVersions)
+			readCalendar(cfg.FeedsPath, feed.String(), calendar, feedVersions)
 		}
 		*versions = allVersions
 	}

+ 39 - 4
server/router.go

@@ -1,6 +1,7 @@
 package server
 
 import (
+	api_structs "notabug.org/apiote/bimba_server/api/structs"
 	"notabug.org/apiote/bimba_server/config"
 	"notabug.org/apiote/bimba_server/traffic"
 	"notabug.org/apiote/bimba_server/traffic/feeds"
@@ -10,8 +11,11 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"os"
 	"strings"
 	"time"
+
+	"git.sr.ht/~sircmpwn/go-bare"
 )
 
 func handleRoot(w http.ResponseWriter, r *http.Request) {
@@ -42,15 +46,24 @@ func handleDepartures(w http.ResponseWriter, r *http.Request, feedName string, c
 	feedTime, _ := time.ParseInLocation("20060102", date, cfg.EnabledFeeds[feedName].GetLocation())
 	for _, v := range versions {
 		if !v.ValidFrom.After(feedTime) && !feedTime.After(v.ValidTill) {
-			versionCode = traffic_structs.Validity(v.ValidFrom.Format("20060102") + "_" + v.ValidTill.Format("20060102"))
+			versionCode = traffic_structs.Validity(v.String())
 		}
 	}
 
-	// todo if versionCode == "" -> error
+	if versionCode == "" {
+		send404(w, r, "date", date)
+		return
+	}
 
 	ix := codeIndex[versionCode]
 	cal := calendar[versionCode]
 
+	stopCodePresent := ix[code]
+	if !stopCodePresent.Present {
+		send404(w, r, "code", string(code))
+		return
+	}
+
 	departures := traffic.GetDepartures(code, cfg.FeedsPath, feedName, versionCode, ix, cal, date, departuresType)
 
 	// this is only temporary
@@ -66,6 +79,21 @@ func handleDepartures(w http.ResponseWriter, r *http.Request, feedName string, c
 
 }
 
+func send404(w http.ResponseWriter, r *http.Request, id, value string) {
+	response := api_structs.ErrorResponse{
+		Success: false,
+		Field:   id,
+		Message: value + " not found as " + id,
+	}
+	b, err := bare.Marshal(&response)
+	if err != nil {
+		w.WriteHeader(500)
+		return
+	}
+	w.WriteHeader(404)
+	w.Write(b)
+}
+
 func Route(cfg config.Config, codeIndexes *traffic_structs.GlobalCodeIndex, calendar *traffic_structs.GlobalCalendar, versions *feeds.GlobalVersions) *http.Server {
 	srv := &http.Server{Addr: ":51354"}
 
@@ -75,10 +103,14 @@ func Route(cfg config.Config, codeIndexes *traffic_structs.GlobalCodeIndex, cale
 		} else {
 			path := strings.Split(r.URL.Path[1:], "/")
 			feedName := path[0]
+			v := *versions
+			if v[feedName] == nil {
+				send404(w, r, "feed", feedName)
+				return
+			}
 			if len(path) == 1 {
 				handleFeed(w, r, feedName)
 			} else {
-				v := *versions
 				ix := *codeIndexes
 				c := *calendar
 				resource := path[1]
@@ -87,6 +119,8 @@ func Route(cfg config.Config, codeIndexes *traffic_structs.GlobalCodeIndex, cale
 					handleStops(w, r, feedName)
 				case "departures":
 					handleDepartures(w, r, feedName, cfg, v[feedName], ix[feedName], c[feedName])
+				default:
+					send404(w, r, "resource", resource)
 				}
 			}
 		}
@@ -94,7 +128,8 @@ func Route(cfg config.Config, codeIndexes *traffic_structs.GlobalCodeIndex, cale
 
 	go func() {
 		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
-			log.Fatalf("ListenAndServe(): %v", err)
+			log.Printf("ListenAndServe(): %v", err)
+			os.Exit(1)
 		}
 	}()
 	return srv

+ 2 - 2
traffic/access.go

@@ -73,7 +73,7 @@ func marshalStopOrder(tripOffset uint, stopOrder int) string {
 }
 
 func GetDepartures(stopCode traffic_structs.ID, dataHome, feedName string, versionCode traffic_structs.Validity, codeIndex traffic_structs.CodeIndex, calendar []traffic_structs.Schedule, date string, departuresType string) []apiStructs.Departure {
-	stopOffset := codeIndex[stopCode]
+	stopCodePresent := codeIndex[stopCode]
 
 	deadzone, _ := time.ParseDuration("-1m")
 	location, _ := time.LoadLocation("Europe/Warsaw") // todo stopLocation ?: feedLocation
@@ -93,7 +93,7 @@ func GetDepartures(stopCode traffic_structs.ID, dataHome, feedName string, versi
 
 	stopsFile, _ := os.Open(filepath.Join(timetableHome, "stops.bare"))
 	defer stopsFile.Close()
-	stopsFile.Seek(int64(stopOffset), 0)
+	stopsFile.Seek(int64(stopCodePresent.Offset), 0)
 	stop := traffic_structs.Stop{}
 	bare.UnmarshalReader(stopsFile, &stop)
 

+ 3 - 13
traffic/convert.go

@@ -635,18 +635,7 @@ func findValidVersions(input ...interface{}) interface{} {
 	sort.Slice(args.downloadedVersions, func(i, j int) bool {
 		return args.downloadedVersions[i].ValidFrom.Before(args.downloadedVersions[j].ValidFrom)
 	})
-	i := 0
-	for _, version := range dateValidVersions {
-		if version.ValidFrom.Before(now) {
-			i++
-		} else {
-			break
-		}
-	}
-	if i > 0 {
-		i = i - 1
-	}
-	validVersions := dateValidVersions[i:]
+	validVersions := feeds.FindValidVersions(dateValidVersions, now)
 
 	missingVersions := []feeds.Version{}
 	for _, version := range validVersions {
@@ -662,7 +651,7 @@ func getGtfsFiles(input ...interface{}) (interface{}, error) {
 	args := input[0].(result)
 	names := []string{}
 	for _, version := range args.missingVersions {
-		name := version.ValidFrom.Format("20060102") + "_" + version.ValidTill.Format("20060102") + ".zip"
+		name := version.String() + ".zip"
 		zipPath := filepath.Join(args.tmpFeedPath, name)
 		url := version.Link
 		response, err := http.Get(url)
@@ -687,6 +676,7 @@ func getGtfsFiles(input ...interface{}) (interface{}, error) {
 func convert(input ...interface{}) (interface{}, error) {
 	args := input[0].(result)
 	for _, gtfsFile := range args.gtfsFilenames {
+		// todo to gott
 		err := file.UnzipGtfs(args.tmpFeedPath, gtfsFile)
 		if err != nil {
 			return gott.Tuple{args}, fmt.Errorf("while unzipping gtfs %s: %w\n", gtfsFile, err)

+ 41 - 0
traffic/feeds/feeds.go

@@ -1,6 +1,8 @@
 package feeds
 
 import (
+	"fmt"
+	"strings"
 	"time"
 )
 
@@ -17,10 +19,49 @@ type Version struct {
 	ValidTill time.Time
 }
 
+func (v Version) String() string {
+	return v.ValidFrom.Format("20060102") + "_" + v.ValidTill.Format("20060102")
+}
+
 func RegisterFeeds() map[string]Feed {
 	return map[string]Feed{
 		"poznan_ztm": ZtmPoznan{},
 	}
 }
 
+func MakeVersion(s string) (Version, error) {
+	version := Version{}
+	versionDates := strings.Split(s, "_")
+	if len(versionDates) != 2 {
+		return version, fmt.Errorf("invalid version string %s, not /.*_.*/", s)
+	}
+	validFrom, err := time.Parse("20060102", versionDates[0])
+	if err != nil {
+		return version, fmt.Errorf("invalid first part in %s: %w", s, err)
+	}
+	validTill, err := time.Parse("20060102", versionDates[1])
+	if err != nil {
+		return version, fmt.Errorf("invalid second part in %s: %w", s, err)
+	}
+	version.ValidFrom = validFrom
+	version.ValidTill = validTill
+	return version, nil
+}
+
+func FindValidVersions(versions []Version, now time.Time) []Version {
+	i := 0
+	for _, version := range versions {
+		if version.ValidFrom.Before(now) {
+			i++
+		} else {
+			break
+		}
+	}
+	if i > 0 {
+		i = i - 1
+	}
+	validVersions := versions[i:]
+	return validVersions
+}
+
 type GlobalVersions map[string][]Version

+ 4 - 13
traffic/feeds/poznan_ztm.go

@@ -113,21 +113,12 @@ func (z ZtmPoznan) GetVersions(date time.Time) ([]Version, error) {
 			}
 		}
 		if validityString != "" && link != "" {
-			validity := strings.Split(validityString, "_")
-			validityFrom, err := time.ParseInLocation("20060102", validity[0], date.Location())
+			version, err := MakeVersion(validityString)
 			if err != nil {
-				return []Version{}, fmt.Errorf("GetVersions: cannot parse valid-from date ‘%s’: %w", validity[0], err)
+				return nil, err
 			}
-			validityTill, err := time.ParseInLocation("20060102", validity[1], date.Location())
-			if err != nil {
-				return []Version{}, fmt.Errorf("GetVersions: cannot parse valid-till date ‘%s’: %w", validity[1], err)
-			}
-
-			versions = append(versions, Version{
-				Link:      "https://ztm.poznan.pl/" + link,
-				ValidFrom: validityFrom,
-				ValidTill: validityTill,
-			})
+			version.Link = "https://ztm.poznan.pl/" + link
+			versions = append(versions, version)
 		}
 	}
 	return versions, nil

+ 4 - 1
traffic/structs/access.go

@@ -33,7 +33,10 @@ func ReadCodeIndex(dataHome, feedName string, validity Validity) CodeIndex {
 	for i := uint64(0); i < num; i++ {
 		k, _ := r.ReadString()
 		v, _ := r.ReadUint()
-		ix[ID(k)] = uint(v)
+		ix[ID(k)] = PresentCode{
+			Present: true,
+			Offset: uint(v),
+		}
 	}
 
 	return ix

+ 5 - 1
traffic/structs/structs.go

@@ -70,7 +70,11 @@ type TrieIndexSegment struct {
 type ID string
 type Validity string // 20060102_20060102
 
-type CodeIndex map[ID]uint
+type PresentCode struct {
+	Present bool
+	Offset uint
+}
+type CodeIndex map[ID]PresentCode
 type FeedCodeIndex map[Validity]CodeIndex
 type GlobalCodeIndex map[string]FeedCodeIndex