tweed
a Twitch RSS feed generator
git clone https://jhhm.no/repos/tweed.git
tweed/store.go
get raw file: store.go
1package main 2 3import ( 4 "context" 5 "crypto/sha1" 6 "encoding/xml" 7 "errors" 8 "fmt" 9 "html" 10 "os" 11 "path/filepath" 12 "strings" 13 "syscall" 14 "time" 15) 16 17type StreamInfo struct { 18 Channel string 19 Title string 20 StreamID string 21 StartedAt time.Time 22} 23 24type RSS struct { 25 XMLName xml.Name `xml:"rss"` 26 Version string `xml:"version,attr"` 27 Channel RSSChannel `xml:"channel"` 28} 29 30type RSSChannel struct { 31 Title string `xml:"title"` 32 Link string `xml:"link"` 33 Description string `xml:"description,omitempty"` 34 LastBuildDate string `xml:"lastBuildDate"` 35 TTL int `xml:"ttl"` 36 Items []RSSItem `xml:"item"` 37} 38 39type RSSItem struct { 40 Title string `xml:"title"` 41 Links []string `xml:"link"` 42 Description string `xml:"description"` 43 Guid string `xml:"guid"` 44 PubDate string `xml:"pubDate"` 45} 46 47func rssFilePath(channel string, cfg *Config) string { 48 return filepath.Join(cfg.DataDir, "feeds", channel+".rss") 49} 50 51func lockDataDir(cfg *Config) (*os.File, error) { 52 dir := cfg.DataDir 53 path := filepath.Join(dir, "tweed.lock") 54 f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) 55 if err != nil { 56 return nil, fmt.Errorf("data dir %s: %w", dir, err) 57 } 58 if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { 59 f.Close() 60 if errors.Is(err, syscall.EWOULDBLOCK) { 61 return nil, fmt.Errorf("another tweed instance is using data dir %s", dir) 62 } 63 return nil, fmt.Errorf("data dir %s: locking %s: %w", dir, path, err) 64 } 65 if err := f.Truncate(0); err != nil { 66 f.Close() 67 return nil, fmt.Errorf("data dir %s: %w", dir, err) 68 } 69 if _, err := fmt.Fprintf(f, "%d\n", os.Getpid()); err != nil { 70 f.Close() 71 return nil, fmt.Errorf("data dir %s: %w", dir, err) 72 } 73 return f, nil 74} 75 76func getPermanentVODURLs(ctx context.Context, cfg *Config, stream *StreamInfo) []string { 77 timestamp := stream.StartedAt.Unix() 78 raw := fmt.Sprintf("%s_%s_%d", stream.Channel, stream.StreamID, timestamp) 79 h := sha1.Sum([]byte(raw)) 80 sha20 := fmt.Sprintf("%x", h)[:20] 81 82 path := fmt.Sprintf("%s_%s/chunked/index-dvr.m3u8", sha20, raw) 83 84 return probeCDNs(ctx, cfg.CDNDomains, path, 3) 85} 86 87func loadRSS(channel string, cfg *Config) (RSS, error) { 88 path := rssFilePath(channel, cfg) 89 data, err := os.ReadFile(path) 90 if err != nil { 91 return RSS{}, err 92 } 93 var rss RSS 94 if err := xml.Unmarshal(data, &rss); err != nil { 95 return RSS{}, err 96 } 97 return rss, nil 98} 99 100func saveRSS(channel string, rss RSS, cfg *Config) error { 101 out, err := xml.MarshalIndent(rss, "", " ") 102 if err != nil { 103 return err 104 } 105 out = append([]byte(xml.Header), out...) 106 107 path := rssFilePath(channel, cfg) 108 dir := filepath.Dir(path) 109 if err := os.Mkdir(dir, 0755); err != nil && !errors.Is(err, os.ErrExist) { 110 return err 111 } 112 tmp, err := os.CreateTemp(filepath.Dir(path), "."+channel+".rss.*") 113 if err != nil { 114 return err 115 } 116 defer os.Remove(tmp.Name()) // clean up on failure 117 118 if _, err := tmp.Write(out); err != nil { 119 tmp.Close() 120 return err 121 } 122 if err := tmp.Sync(); err != nil { 123 tmp.Close() 124 return err 125 } 126 if err := tmp.Close(); err != nil { 127 return err 128 } 129 if err := os.Chmod(tmp.Name(), 0644); err != nil { 130 return err 131 } 132 return os.Rename(tmp.Name(), path) 133} 134 135func newRSS(cfg *Config, channel string) RSS { 136 return RSS{ 137 Version: "2.0", 138 Channel: RSSChannel{ 139 Title: channel + "'s Twitch streams", 140 Link: "https://twitch.tv/" + channel, // NOTE: should maybe be tweed link 141 Description: "A feed of " + channel + "'s streams.", 142 LastBuildDate: time.Now().Format(time.RFC1123Z), // TODO: what time format to use? 143 TTL: cfg.TTLMinutes, 144 }, 145 } 146} 147 148func initEmptyFeeds(cfg *Config) error { 149 for _, c := range cfg.Channels { 150 if _, err := os.Stat(rssFilePath(c, cfg)); errors.Is(err, os.ErrNotExist) { 151 rss := newRSS(cfg, c) 152 if err := saveRSS(c, rss, cfg); err != nil { 153 return err 154 } 155 } 156 } 157 return nil 158} 159 160func makeRSSDescription(cdns []string) string { 161 var sb strings.Builder 162 // TODO: add some message like 'x whent live at y, stream id: z' 163 sb.WriteString("<h2>CDN links:</h2>\n") 164 sb.WriteString("<ul>\n") 165 for _, u := range cdns { 166 escaped := html.EscapeString(u) 167 sb.WriteString(fmt.Sprintf( 168 ` <li><a href="%s" target="_blank" rel="noopener noreferrer">%s</a></li>`+"\n", 169 escaped, escaped, 170 )) 171 } 172 sb.WriteString("</ul>\n") 173 174 return sb.String() 175} 176 177func updateRSSWithNewStream(ctx context.Context, cfg *Config, stream *StreamInfo) error { 178 rss, err := loadRSS(stream.Channel, cfg) 179 if errors.Is(err, os.ErrNotExist) { 180 rss = newRSS(cfg, stream.Channel) 181 } else if err != nil { 182 return fmt.Errorf("loading feed for %s: %w", stream.Channel, err) 183 } 184 for _, item := range rss.Channel.Items { 185 if item.Guid == stream.StreamID { 186 logInfo("stream %s already in feed for %s", stream.StreamID, stream.Channel) 187 return nil 188 } 189 } 190 191 channelUrl := "https://twitch.tv/" + stream.Channel 192 193 links := getPermanentVODURLs(ctx, cfg, stream) 194 if ctx.Err() != nil { 195 return ctx.Err() 196 } 197 if len(links) == 0 { 198 links = []string{channelUrl} 199 } 200 pub := stream.StartedAt.Format(time.RFC822Z) 201 202 item := RSSItem{ 203 Title: stream.Title, 204 Links: links, 205 Description: makeRSSDescription(links), 206 Guid: stream.StreamID, 207 PubDate: pub, 208 } 209 210 rss.Channel.Items = append([]RSSItem{item}, rss.Channel.Items...) 211 rss.Channel.LastBuildDate = time.Now().Format(time.RFC1123Z) 212 rss.Channel.TTL = cfg.TTLMinutes 213 214 logInfo("storing stream '%s' from %s, %d cdn links", stream.Title, stream.Channel, len(links)) 215 216 return saveRSS(stream.Channel, rss, cfg) 217}