@@ -12,9 +12,9 @@ without getting bogged down by the complexity of concurrency.
1212- ** Error Handling** : provides a structured way to handle errors in concurrent apps
1313- ** Streaming** : handles real-time data streams or large datasets with a minimal memory footprint
1414- ** Order Preservation** : offers functions that preserve the original order of data, while still allowing for concurrent processing
15- - ** Functional Programming ** : based on functional programming concepts, making operations like map, filter, flatMap and others available for channel-based workflows
15+ - ** Efficient Resource Use ** : the number of goroutines and allocations does not depend on data size
1616- ** Generic** : all operations are type-safe and can be used with any data type
17-
17+ - ** Functional Programming ** : based on functional programming concepts, making operations like map, filter, flatMap and others available for channel-based workflows
1818
1919## Installation
2020``` bash
@@ -33,59 +33,68 @@ type KV struct {
3333 Value string
3434}
3535
36- func printValues (ctx context .Context , urls []string ) error {
37- ctx , cancel := context.WithCancel (ctx)
38- defer cancel () // In case of error or early exit, this ensures all http and redis operations are canceled
39-
40- // Convert URLs into a channel
41- urlsChan := echans.FromSlice (urls)
42-
43- // Fetch and stream keys from each URL concurrently
44- keys := echans.FlatMap (urlsChan, 10 , func (url string ) <- chan echans.Try [string ] {
45- return streamKeys (ctx, url)
46- })
47-
48- // Exclude any empty keys from the stream
49- keys = echans.Filter (keys, 5 , func (key string ) (bool , error ) {
50- return key != " " nil
51- })
52-
53- // Organize keys into manageable batches of 10 for bulk operations
54- keyBatches := echans.Batch (keys, 10 , 1 *time.Second )
55-
56- // Fetch values from Redis for each batch of keys
57- resultBatches := echans.Map (keyBatches, 5 , func (keys []string ) ([]KV, error ) {
58- values , err := redisMGet (ctx, keys...)
59- if err != nil {
60- return nil , err
61- }
62-
63- results := make ([]KV, len (keys))
64- for i , key := range keys {
65- results[i] = KV{Key: key, Value: values[i]}
66- }
67-
68- return results, nil
69- })
70-
71- // Convert batches back to a single items for final processing
72- results := echans.Unbatch (resultBatches)
73-
74- // Exclude any empty values from the stream
75- results = echans.Filter (results, 5 , func (kv KV) (bool , error ) {
76- return kv.Value != " <nil>" nil
77- })
78-
79- // Iterate over each key-value pair and print
80- cnt := 0
81- err := echans.ForEach (results, 1 , func (kv KV) error {
82- fmt.Println (kv.Key , " =>" Value )
83- cnt++
84- return nil
85- })
86- fmt.Println (" Total keys:"
87-
88- return err
36+
37+ func printValuesFromRedis (ctx context .Context , urls []string ) error {
38+ ctx , cancel := context.WithCancel (ctx)
39+ defer cancel () // In case of error, this ensures all http and redis operations are canceled
40+
41+ // Convert urls into a channel
42+ urlsChan := rill.WrapSlice (urls)
43+
44+ // Fetch and stream keys from each URL concurrently
45+ keys := rill.FlatMap (urlsChan, 10 , func (url string ) <- chan rill.Try [string ] {
46+ return streamKeys (ctx, url)
47+ })
48+
49+ // Exclude any empty keys from the stream
50+ keys = rill.Filter (keys, 5 , func (key string ) (bool , error ) {
51+ return key != " " nil
52+ })
53+
54+ // Organize keys into manageable batches of 10 for bulk operations
55+ keyBatches := rill.Batch (keys, 10 , 1 *time.Second )
56+
57+ // Fetch values from Redis for each batch of keys
58+ resultBatches := rill.Map (keyBatches, 5 , func (keys []string ) ([]KV, error ) {
59+ values , err := redisMGet (ctx, keys...)
60+ if err != nil {
61+ return nil , err
62+ }
63+
64+ results := make ([]KV, len (keys))
65+ for i , key := range keys {
66+ results[i] = KV{Key: key, Value: values[i]}
67+ }
68+
69+ return results, nil
70+ })
71+
72+ // Convert batches back to a single items for final processing
73+ results := rill.Unbatch (resultBatches)
74+
75+ // Exclude any empty values from the stream
76+ results = rill.Filter (results, 5 , func (kv KV) (bool , error ) {
77+ return kv.Value != " <nil>" nil
78+ })
79+
80+ // Iterate over each key-value pair and print
81+ cnt := 0
82+ err := rill.ForEach (results, 1 , func (kv KV) error {
83+ fmt.Println (kv.Key , " =>" Value )
84+ cnt++
85+ return nil
86+ })
87+ if err != nil {
88+ return err
89+ }
90+
91+ fmt.Println (" Total keys:"
92+ return nil
93+ }
94+
95+ // streamKeys reads a file from the given URL line by line and returns a channel of lines/keys
96+ func streamKeys (ctx context .Context , url string ) <-chan rill .Try [string ] {
97+ // ...
8998}
9099
91100
@@ -96,9 +105,9 @@ func printValues(ctx context.Context, urls []string) error {
96105
97106## Design philosophy
98107At the heart of rill lies a simple yet powerful concept: operating on channels of wrapped values, encapsulated by the Try structure.
99- Such channels can be created manually or through utilities like ** FromSlice ** , ** Wrap ** , and ** WrapAsync ** , and then transformed via operations
108+ Such channels can be created manually or through utilities like ** WrapSlice ** or ** WrapChan ** , and then transformed via operations
100109such as ** Map** , ** Filter** , ** FlatMap** and others. Finally when all processing stages are completed, the data can be consumed by
101- ** ForEach** , ** ToSlice ** or manually by iterating over the resulting channel.
110+ ** ForEach** , ** UnwrapToSlice ** or manually by iterating over the resulting channel.
102111
103112
104113
@@ -151,39 +160,44 @@ type Measurement struct {
151160}
152161
153162func printTemperatureMovements (ctx context .Context , city string , startDate , endDate time .Time ) error {
154- ctx , cancel := context.WithCancel (ctx)
155- defer cancel () // In case of error or early exit, this ensures all http are canceled
156-
157- // Make a channel that emits all the days between startDate and endDate
158- days := make (chan echans.Try [time.Time ])
159- go func () {
160- defer close (days)
161- for date := startDate; date.Before (endDate); date = date.AddDate (0 , 0 , 1 ) {
162- days <- echans.Try [time.Time ]{V: date}
163- }
164- }()
165-
166- // Download the temperature for each day in parallel and in order
167- measurements := echans.OrderedMap (days, 10 , func (date time.Time ) (Measurement, error ) {
168- temp , err := getTemperature (ctx, city, date)
169- return Measurement{Date: date, Temp: temp}, err
170- })
171-
172- // Calculate the temperature movements. Use a single goroutine
173- prev := Measurement{Temp: math.NaN ()}
174- measurements = echans.OrderedMap (measurements, 1 , func (m Measurement) (Measurement, error ) {
175- m.Movement = m.Temp - prev.Temp
176- prev = m
177- return m, nil
178- })
179-
180- // Iterate over the measurements and print the movements
181- err := echans.ForEach (measurements, 1 , func (m Measurement) error {
182- fmt.Printf (" %s : %.1f °C (movement %+.1f °C)\n " Date .Format (" 2006-01-02" Temp , m.Movement )
183- prev = m
184- return nil
185- })
186-
187- return err
163+ ctx , cancel := context.WithCancel (ctx)
164+ defer cancel () // In case of error, this ensures all pending operations are canceled
165+
166+ // Make a channel that emits all the days between startDate and endDate
167+ days := make (chan rill.Try [time.Time ])
168+ go func () {
169+ defer close (days)
170+ for date := startDate; date.Before (endDate); date = date.AddDate (0 , 0 , 1 ) {
171+ days <- rill.Wrap (date, nil )
172+ }
173+ }()
174+
175+ // Download the temperature for each day in parallel and in order
176+ measurements := rill.OrderedMap (days, 10 , func (date time.Time ) (Measurement, error ) {
177+ temp , err := getTemperature (ctx, city, date)
178+ return Measurement{Date: date, Temp: temp}, err
179+ })
180+
181+ // Calculate the temperature movements. Use a single goroutine
182+ prev := Measurement{Temp: math.NaN ()}
183+ measurements = rill.OrderedMap (measurements, 1 , func (m Measurement) (Measurement, error ) {
184+ m.Movement = m.Temp - prev.Temp
185+ prev = m
186+ return m, nil
187+ })
188+
189+ // Iterate over the measurements and print the movements
190+ err := rill.ForEach (measurements, 1 , func (m Measurement) error {
191+ fmt.Printf (" %s : %.1f °C (movement %+.1f °C)\n " Date .Format (" 2006-01-02" Temp , m.Movement )
192+ prev = m
193+ return nil
194+ })
195+
196+ return err
197+ }
198+
199+ // getTemperature does a network request to fetch the temperature for a given city and date.
200+ func getTemperature (ctx context .Context , city string , date time .Time ) (float64 , error ) {
201+ // ...
188202}
189203```
0 commit comments