Update parsing to the new collection dates page

.devcontainer/devcontainer.json

@@ -0,0 +1,32 @@
+{
+  "name": "aklapi-go-dev",
+  "image": "mcr.microsoft.com/devcontainers/go:1.24-bookworm",
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {},
+    "ghcr.io/devcontainers/features/git:1": {}
+  },
+  "forwardPorts": [8080],
+  "portsAttributes": {
+    "8080": {
+      "label": "aklapi",
+      "protocol": "http",
+      "onAutoForward": "notify"
+    }
+  },
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "go.useLanguageServer": true,
+        "go.toolsManagement.autoUpdate": true,
+        "go.formatTool": "goimports",
+        "editor.formatOnSave": true,
+        "go.lintTool": "golangci-lint"
+      },
+      "extensions": [
+        "golang.go"
+      ]
+    }
+  },
+  "runArgs": ["--init"],
+  "containerUser": "vscode"
+}

README.md

@@ -18,15 +18,15 @@ Full list of available endpoints, for detailed description see below.
 
 Two endpoints so far, both accepting `addr` parameter.
 
-* `/api/v1/rr/` - rubbish and recycling, returns the JSON of the following format:
+* `/api/v1/rr` - rubbish and recycling, returns the JSON of the following format:
 
       {
           "rubbish": "2020-02-25",
           "recycle": "2020-02-25",
           "address": "Britomart, CBD"
       }
 
-* `/api/v1/rrext/` - extended rubbish and recycling.  Returns the JSON in the following format:
+* `/api/v1/rrext` - extended rubbish and recycling.  Returns the JSON in the following format:
 
       {
           "Collections": [
@@ -53,7 +53,7 @@ Two endpoints so far, both accepting `addr` parameter.
 Example:
 
 ```sh
-$ curl --location --request GET 'https://<server>/api/v1/rr/?addr=500%20Queen%20Street'
+$ curl --location --request GET 'https://<server>/api/v1/rr?addr=500%20Queen%20Street'
 {"rubbish":"2020-02-24","recycle":"2020-02-24","address":"500 Queen Street, Auckland Central"}
 ```
 
@@ -63,29 +63,21 @@ Assuming your aklapi API server running on localhost:5010, add the following
 to your `configuration.yaml`:
 
 ```yaml
-sensor:
-  - platform: rest
-    resource: "http://localhost:5010/api/v1/rr/?addr=xx"
-    name: Recycle
-    scan_interval: 300
-    value_template: "{{ value_json.recycle }}"
+rest:
+  - resource: http://localhost:5010/api/v1/rr?addr=xx
     method: GET
-    unique_id: recycle_date
-
-  - platform: rest
-    resource: "http://localhost:5010/api/v1/rr/?addr=xx"
-    name: Food Scraps
     scan_interval: 300
-    value_template: "{{ value_json.foodscraps }}"
-    method: GET
-    unique_id: foodscraps_date
-
-  - platform: rest
-    resource: "http://localhost:5010/api/v1/rr/?addr=xx"
-    name: Rubbish
-    scan_interval: 300
-    value_template: "{{ value_json.rubbish }}"
-    method: GET
-    unique_id: rubbish_date
-
+    sensor:
+      - name: Recycle
+        value_template: "{{ value_json.recycle }}"
+        device_class: date
+        unique_id: recycle_date
+      - name: Food Scraps
+        value_template: "{{ value_json.foodscraps }}"
+        device_class: date
+        unique_id: foodscraps_date
+      - name: Rubbish
+        value_template: "{{ value_json.rubbish }}"
+        device_class: date
+        unique_id: rubbish_date
 ```

cmd/aklapi/main.go

@@ -24,9 +24,9 @@ const (
 	root              = "/"
 	apiHealthcheck    = "/healthcheck"
 	apiRoot           = "/api/v1"
-	apiAddr           = apiRoot + "/addr/"
-	apiRubbishRecycle = apiRoot + "/rr/"
-	apiRRExt          = apiRoot + "/rrext/"
+	apiAddr           = apiRoot + "/addr"
+	apiRubbishRecycle = apiRoot + "/rr"
+	apiRRExt          = apiRoot + "/rrext"
 )
 
 func init() {

rubbish.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
@@ -15,12 +16,12 @@ import (
 var NoCache = false
 
 const (
-	dateLayout = "Monday 2 January"
+	dateLayout = "Monday, 2 January"
 )
 
 var (
 	// defined as a variable so it can be overridden in tests.
-	collectionDayURI = `https://www.aucklandcouncil.govt.nz/rubbish-recycling/rubbish-recycling-collections/Pages/collection-day-detail.aspx?an=%s`
+	collectionDayURI = `https://new.aucklandcouncil.govt.nz/en/rubbish-recycling/rubbish-recycling-collections/rubbish-recycling-collection-days/%s.html`
 )
 
 var errSkip = errors.New("skip this date")
@@ -142,16 +143,22 @@ type refuseParser struct {
 
 // Parse parses the auckland council rubbish webpage.
 func (p *refuseParser) parse(r io.Reader) ([]RubbishCollection, error) {
-	const datesSection = "#ctl00_SPWebPartManager1_g_dfe289d2_6a8a_414d_a384_fc25a0db9a6d_ctl00_pnlHouseholdBlock2"
-	p.detail = make([]RubbishCollection, 3)
+	const scheduledCardSelector = ".acpl-schedule-card"
+	p.detail = make([]RubbishCollection, 0, 3)
 	doc, err := goquery.NewDocumentFromReader(r)
 	if err != nil {
 		return nil, err
 	}
-	_ = doc.Find(datesSection).
-		Children().
-		Slice(1, 4).
-		Each(p.parseLinks) // p.parseLinks populates p.detail
+
+	doc.Find(scheduledCardSelector).Each(func(i int, card *goquery.Selection) {
+		// Only parse Household collection card for now, not Commercial
+		cardTitle := strings.TrimSpace(card.Find("h4.card-title").Text())
+		if !strings.Contains(cardTitle, "Household collection") {
+			return
+		}
+		p.parseScheduleCard(card)
+	})
+
 	for i := range p.detail {
 		if err := (&p.detail[i]).parseDate(); err != nil {
 			if err == errSkip {
@@ -167,31 +174,28 @@ func (p *refuseParser) parse(r io.Reader) ([]RubbishCollection, error) {
 	return p.detail, p.Err
 }
 
-// parseLinks parses the links within selection and populates p.detail.
-func (p *refuseParser) parseLinks(el int, sel *goquery.Selection) {
-	sel.Children().Children().Each(func(n int, sel *goquery.Selection) {
-		switch n {
-		case 0:
-			attr, found := sel.Attr("class")
-			if !found {
-				return
-			}
-			if attr == "icon-rubbish" {
-				p.detail[el].Rubbish = true
-			} else if attr == "icon-food-waste" {
-				p.detail[el].FoodScraps = true
-			} else if attr == "icon-recycle" {
-				p.detail[el].Recycle = true
-			} else {
-				p.Err = fmt.Errorf("parse error: sel.Text = %q, el = %d, n = %d", sel.Text(), el, n)
-			}
-		default:
-			if dow.FindString(sel.Text()) == "" {
-				log.Println("unable to detect day of week")
-				return
-			}
-			p.detail[el].Day = sel.Text()
+// parseScheduleCard parses a schedule card and extracts collection information.
+func (p *refuseParser) parseScheduleCard(card *goquery.Selection) {
+	// Each collection line is a <p class="mb-0 lead"> containing a span.acpl-icon-with-attribute.left
+	card.Find("p.mb-0.lead span.acpl-icon-with-attribute.left").Each(func(i int, span *goquery.Selection) {
+		icon := span.Find("i.acpl-icon")
+		date := strings.TrimSpace(span.Find("b").First().Text())
+		if date == "" { // skip empty (e.g. food scraps absent)
+			return
+		}
+		var rc RubbishCollection
+		rc.Day = date
+		if icon.HasClass("rubbish") {
+			rc.Rubbish = true
+		} else if icon.HasClass("recycle") {
+			rc.Recycle = true
+		} else if icon.HasClass("food-waste") {
+			rc.FoodScraps = true
+		} else {
+			// Unknown icon type; ignore
+			return
 		}
+		p.detail = append(p.detail, rc)
 	})
 }
 

rubbish_test.go

@@ -14,8 +14,8 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-//go:generate curl -L https://www.aucklandcouncil.govt.nz/rubbish-recycling/rubbish-recycling-collections/Pages/collection-day-detail.aspx?an=12342478585 -o test_assets/500-queen-street.html
-//go:generate curl -L https://www.aucklandcouncil.govt.nz/rubbish-recycling/rubbish-recycling-collections/Pages/collection-day-detail.aspx?an=12341511281 -o test_assets/1-luanda-drive.html
+//go:generate curl -L https://new.aucklandcouncil.govt.nz/en/rubbish-recycling/rubbish-recycling-collections/rubbish-recycling-collection-days/12342478585.html -o test_assets/500-queen-street.html
+//go:generate curl -L https://new.aucklandcouncil.govt.nz/en/rubbish-recycling/rubbish-recycling-collections/rubbish-recycling-collection-days/12341511281.html -o test_assets/1-luanda-drive.html
 
 // Test data, run go:generate to update, then update dates in tests
 // accordingly.
@@ -42,22 +42,22 @@ func Test_parse(t *testing.T) {
 			&CollectionDayDetailResult{
 				Collections: []RubbishCollection{
 					{
-						Day:        "Tuesday 27 August",
-						Date:       adjustYear(time.Date(0, 8, 27, 0, 0, 0, 0, defaultLoc)),
+						Day:        "Tuesday, 18 November",
+						Date:       adjustYear(time.Date(0, 11, 18, 0, 0, 0, 0, defaultLoc)),
 						Rubbish:    true,
 						Recycle:    false,
 						FoodScraps: false,
 					},
 					{
-						Day:        "Tuesday 27 August",
-						Date:       adjustYear(time.Date(0, 8, 27, 0, 0, 0, 0, defaultLoc)),
+						Day:        "Tuesday, 18 November",
+						Date:       adjustYear(time.Date(0, 11, 18, 0, 0, 0, 0, defaultLoc)),
 						Rubbish:    false,
 						Recycle:    false,
 						FoodScraps: true,
 					},
 					{
-						Day:        "Tuesday 3 September",
-						Date:       adjustYear(time.Date(0, 9, 3, 0, 0, 0, 0, defaultLoc)),
+						Day:        "Tuesday, 25 November",
+						Date:       adjustYear(time.Date(0, 11, 25, 0, 0, 0, 0, defaultLoc)),
 						Rubbish:    false,
 						Recycle:    true,
 						FoodScraps: false,
@@ -71,14 +71,14 @@ func Test_parse(t *testing.T) {
 			&CollectionDayDetailResult{
 				Collections: []RubbishCollection{
 					{
-						Day:     "Thursday 22 August",
-						Date:    adjustYear(time.Date(0, 8, 22, 0, 0, 0, 0, defaultLoc)),
+						Day:     "Saturday, 15 November",
+						Date:    adjustYear(time.Date(0, 11, 15, 0, 0, 0, 0, defaultLoc)),
 						Rubbish: true,
 						Recycle: false,
 					},
 					{
-						Day:     "Thursday 22 August",
-						Date:    adjustYear(time.Date(0, 8, 22, 0, 0, 0, 0, defaultLoc)),
+						Day:     "Saturday, 15 November",
+						Date:    adjustYear(time.Date(0, 11, 15, 0, 0, 0, 0, defaultLoc)),
 						Rubbish: false,
 						Recycle: true,
 					},
@@ -116,22 +116,22 @@ func TestCollectionDayDetail(t *testing.T) {
 			&CollectionDayDetailResult{
 				Collections: []RubbishCollection{
 					{
-						Day:        "Tuesday 27 August",
-						Date:       adjustYear(time.Date(0, 8, 27, 0, 0, 0, 0, defaultLoc)),
+						Day:        "Tuesday, 18 November",
+						Date:       adjustYear(time.Date(0, 11, 18, 0, 0, 0, 0, defaultLoc)),
 						Rubbish:    true,
 						Recycle:    false,
 						FoodScraps: false,
 					},
 					{
-						Day:        "Tuesday 27 August",
-						Date:       adjustYear(time.Date(0, 8, 27, 0, 0, 0, 0, defaultLoc)),
+						Day:        "Tuesday, 18 November",
+						Date:       adjustYear(time.Date(0, 11, 18, 0, 0, 0, 0, defaultLoc)),
 						Rubbish:    false,
 						Recycle:    false,
 						FoodScraps: true,
 					},
 					{
-						Day:        "Tuesday 3 September",
-						Date:       adjustYear(time.Date(0, 9, 3, 0, 0, 0, 0, defaultLoc)),
+						Day:        "Tuesday, 25 November",
+						Date:       adjustYear(time.Date(0, 11, 25, 0, 0, 0, 0, defaultLoc)),
 						Rubbish:    false,
 						Recycle:    true,
 						FoodScraps: false,
@@ -152,7 +152,7 @@ func TestCollectionDayDetail(t *testing.T) {
 			oldAddrURI := addrURI
 			oldcollectionDayURI := collectionDayURI
 			defer func() { addrURI = oldAddrURI; collectionDayURI = oldcollectionDayURI }()
-			addrURI = tt.testSrv.URL + "/addr/"
+			addrURI = tt.testSrv.URL + "/addr"
 			collectionDayURI = tt.testSrv.URL + "/rubbish/?an=%s"
 			got, err := CollectionDayDetail(tt.args.addr)
 			if (err != nil) != tt.wantErr {
@@ -264,7 +264,7 @@ func TestRubbishCollection_parseDate(t *testing.T) {
 		fields  fields
 		wantErr bool
 	}{
-		{"some date", fields{Day: "Monday 16 September"}, false},
+		{"some date", fields{Day: "Monday, 16 September"}, false},
 		{"invalid date", fields{Day: "16 September"}, true},
 	}
 	for _, tt := range tests {
@@ -284,7 +284,7 @@ func TestRubbishCollection_parseDate(t *testing.T) {
 
 func testMux() http.Handler {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/addr/", func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc("/addr", func(w http.ResponseWriter, r *http.Request) {
 		data, err := json.Marshal(AddrResponse{*testAddr})
 		if err != nil {
 			panic(err)