Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Asynchronous M3U parsing directly from URL #6

Merged
merged 33 commits into from
Mar 2, 2024
Merged
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
4234d82
add context for graceful shutdown
sonroyaalmerol Mar 1, 2024
f55227c
add basic m3u tests
sonroyaalmerol Mar 1, 2024
efbf65d
remove strings import
sonroyaalmerol Mar 1, 2024
7be1087
fix expected stream url
sonroyaalmerol Mar 1, 2024
56e273d
import fmt
sonroyaalmerol Mar 1, 2024
7400a50
remove newline from expected content
sonroyaalmerol Mar 1, 2024
38fcec5
individualize sql query functions
sonroyaalmerol Mar 2, 2024
bc35489
clean up m3u package and directly connect to database
sonroyaalmerol Mar 2, 2024
c21f25a
comment out to be fixed first
sonroyaalmerol Mar 2, 2024
9df85c2
do not insert channels without titles
sonroyaalmerol Mar 2, 2024
424dde6
fix mp4_handler.go
sonroyaalmerol Mar 2, 2024
8dcfd27
fix database_test
sonroyaalmerol Mar 2, 2024
c261741
add title to insertstream error log
sonroyaalmerol Mar 2, 2024
5394e58
separate insert stream and insert url
sonroyaalmerol Mar 2, 2024
599c990
add rows next
sonroyaalmerol Mar 2, 2024
1f8c874
add more log info test
sonroyaalmerol Mar 2, 2024
6276f5d
add more log info test
sonroyaalmerol Mar 2, 2024
4002bc4
fix test error
sonroyaalmerol Mar 2, 2024
eb38c01
switch to directly accessing url
sonroyaalmerol Mar 2, 2024
2dcd36b
switch to directly accessing url
sonroyaalmerol Mar 2, 2024
db38a94
clean up downloader.go
sonroyaalmerol Mar 2, 2024
ef963ea
add USER_AGENT env var
sonroyaalmerol Mar 2, 2024
3e22d1b
make m3u parsing asynchronous
sonroyaalmerol Mar 2, 2024
f2fc16a
improve logging messages
sonroyaalmerol Mar 2, 2024
5a9a265
fix formatting
sonroyaalmerol Mar 2, 2024
8db9d85
add hadolint and golanci-lint
sonroyaalmerol Mar 2, 2024
0658fbe
fix dockerfile lint issues
sonroyaalmerol Mar 2, 2024
8c5aff9
fix golinting issues
sonroyaalmerol Mar 2, 2024
dbe8f6c
fix linting issues
sonroyaalmerol Mar 2, 2024
e233cd5
add line number to golangci-lint
sonroyaalmerol Mar 2, 2024
b5a2b3b
fix golint issue
sonroyaalmerol Mar 2, 2024
37d161d
only rollback on err
sonroyaalmerol Mar 2, 2024
d1f7a9c
fix #EXTVLCOPT
sonroyaalmerol Mar 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
switch to directly accessing url
sonroyaalmerol committed Mar 2, 2024
commit eb38c012ac6925c99542788809c0335b42fd9b09
75 changes: 44 additions & 31 deletions m3u/m3u_test.go
Original file line number Diff line number Diff line change
@@ -71,8 +71,26 @@ func TestGenerateM3UContent(t *testing.T) {
}
}

// TestParseM3UFile tests the parseM3UFile function.
func TestParseM3UFile(t *testing.T) {
func TestParseM3UFromURL(t *testing.T) {
testM3UContent := `
#EXTM3U
#EXTINF:-1 tvg-id="bbc1" tvg-name="BBC One" group-title="UK",BBC One
http://example.com/bbc1
#EXTINF:-1 tvg-id="bbc2" tvg-name="BBC Two" group-title="UK",BBC Two
http://example.com/bbc2
#EXTINF:-1 tvg-id="cnn" tvg-name="CNN International" group-title="News",CNN
http://example.com/cnn
#EXTVLCOPT:logo=http://example.com/bbc_logo.png
#EXTINF:-1 tvg-id="fox" tvg-name="FOX" group-title="Entertainment",FOX
http://example.com/fox
`
// Create a mock HTTP server
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/octet-stream")
fmt.Fprintln(w, testM3UContent)
}))
defer mockServer.Close()

sqliteDBPath := filepath.Join(".", "data", "database.sqlite")

// Test InitializeSQLite and check if the database file exists
@@ -82,40 +100,36 @@ func TestParseM3UFile(t *testing.T) {
}
defer os.Remove(sqliteDBPath) // Cleanup the database file after the test

// Create a temporary test file
tempFile, err := os.CreateTemp("", "test.m3u")
if err != nil {
t.Fatalf("Failed to create temporary file: %v", err)
}
defer os.Remove(tempFile.Name()) // Clean up

// Write test data to the temporary file
testData := []byte(`
#EXTINF:-1 tvg-id="channel1" tvg-name="Channel 1" group-title="Group 1" tvg-logo="logo1.jpg",Channel 1
http://example.com/channel1
#EXTINF:-1 tvg-id="channel2" tvg-name="Channel 2" group-title="Group 2" tvg-logo="logo2.jpg",Channel 2
http://example.com/channel2
#EXTVLCOPT:logo=http://example.com/logo3.jpg
http://example.com/channel3
`)
if _, err := tempFile.Write(testData); err != nil {
t.Fatalf("Failed to write to temporary file: %v", err)
}
tempFile.Close() // Close the file before parsing

// Call the function with the temporary file
err = parseM3UFile(tempFile.Name(), 0)
// Test the parseM3UFromURL function with the mock server URL
err = parseM3UFromURL(mockServer.URL, 0)
if err != nil {
t.Fatalf("Error parsing M3U file: %v", err)
t.Errorf("Error parsing M3U from URL: %v", err)
}

// Check the expected content in the database
// Verify expected values
expectedStreams := []database.StreamInfo{
{URLs: []database.StreamURL{{Content: "http://example.com/channel1", M3UIndex: 0}}, TvgID: "channel1", Title: "Channel 1", Group: "Group 1", LogoURL: "logo1.jpg"},
{URLs: []database.StreamURL{{Content: "http://example.com/channel2", M3UIndex: 0}}, TvgID: "channel2", Title: "Channel 2", Group: "Group 2", LogoURL: "logo2.jpg"},
{Title: "BBC One", TvgID: "bbc1", Group: "UK", URLs: []database.StreamURL{
{
Content: "http://example.com/bbc1",
},
}},
{Title: "BBC Two", TvgID: "bbc2", Group: "UK", URLs: []database.StreamURL{
{
Content: "http://example.com/bbc2",
},
}},
{Title: "CNN International", TvgID: "cnn", Group: "News", URLs: []database.StreamURL{
{
Content: "http://example.com/cnn",
},
}},
{Title: "FOX", TvgID: "fox", Group: "Entertainment", URLs: []database.StreamURL{
{
Content: "http://example.com/fox",
},
}},
}

// Retrieve streams from the database
storedStreams, err := database.GetStreams()
if err != nil {
t.Fatalf("Error retrieving streams from database: %v", err)
@@ -143,7 +157,6 @@ http://example.com/channel3
}
}


// streamInfoEqual checks if two StreamInfo objects are equal.
func streamInfoEqual(a, b database.StreamInfo) bool {
if a.TvgID != b.TvgID || a.Title != b.Title || a.Group != b.Group || a.LogoURL != b.LogoURL || len(a.URLs) != len(b.URLs) {
14 changes: 7 additions & 7 deletions m3u/parser.go
Original file line number Diff line number Diff line change
@@ -3,23 +3,23 @@ package m3u
import (
"bufio"
"fmt"
"os"
"net/http"
"regexp"
"strings"

"m3u-stream-merger/database"
)

func parseM3UFile(filePath string, m3uIndex int) (error) {
fmt.Printf("Parsing: %s\n", filePath)
func parseM3UFile(m3uURL string, m3uIndex int) (error) {
fmt.Printf("Parsing M3U from URL: %s\n", m3uURL)

file, err := os.Open(filePath)
resp, err := http.Get(m3uURL)
if err != nil {
return fmt.Errorf("Open error: %v", err)
return fmt.Errorf("HTTP GET error: %v", err)
}
defer file.Close()
defer resp.Body.Close()

scanner := bufio.NewScanner(file)
scanner := bufio.NewScanner(resp.Body)

var currentStream database.StreamInfo