Skip to content

Commit 8d8b47e

Browse files
author
Marvin Zhang
committed
refactor: streamline file service retrieval and enhance spider template handling
- Replaced direct calls to getBaseFileFsSvc with a new method fs.GetBaseFileFsSvc in base_file.go for improved clarity and maintainability. - Introduced SpiderTemplateService interface and implemented registry service for managing spider templates, enhancing template handling in the spider controller. - Added template-related fields to the Spider model to support template functionality. - Created utility functions for string case conversions in utils/string.go to facilitate consistent formatting across the codebase. - Updated environment configuration to retrieve the Python path dynamically, improving flexibility in the task runner's setup.
1 parent f5d9ccf commit 8d8b47e

File tree

9 files changed

+110
-28
lines changed

9 files changed

+110
-28
lines changed

‎core/controllers/base_file.go

Lines changed: 10 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,16 @@ import (
44
"errors"
55
"fmt"
66
"github.com/crawlab-team/crawlab/core/fs"
7-
"github.com/crawlab-team/crawlab/core/interfaces"
8-
"github.com/crawlab-team/crawlab/core/utils"
97
"github.com/gin-gonic/gin"
108
"io"
119
"os"
12-
"path/filepath"
1310
"sync"
1411
)
1512

1613
func GetBaseFileListDir(rootPath string, c *gin.Context) {
1714
path := c.Query("path")
1815

19-
fsSvc, err := getBaseFileFsSvc(rootPath)
16+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
2017
if err != nil {
2118
HandleErrorBadRequest(c, err)
2219
return
@@ -36,7 +33,7 @@ func GetBaseFileListDir(rootPath string, c *gin.Context) {
3633
func GetBaseFileFile(rootPath string, c *gin.Context) {
3734
path := c.Query("path")
3835

39-
fsSvc, err := getBaseFileFsSvc(rootPath)
36+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
4037
if err != nil {
4138
HandleErrorBadRequest(c, err)
4239
return
@@ -54,7 +51,7 @@ func GetBaseFileFile(rootPath string, c *gin.Context) {
5451
func GetBaseFileFileInfo(rootPath string, c *gin.Context) {
5552
path := c.Query("path")
5653

57-
fsSvc, err := getBaseFileFsSvc(rootPath)
54+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
5855
if err != nil {
5956
HandleErrorBadRequest(c, err)
6057
return
@@ -70,7 +67,7 @@ func GetBaseFileFileInfo(rootPath string, c *gin.Context) {
7067
}
7168

7269
func PostBaseFileSaveFile(rootPath string, c *gin.Context) {
73-
fsSvc, err := getBaseFileFsSvc(rootPath)
70+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
7471
if err != nil {
7572
HandleErrorInternalServerError(c, err)
7673
return
@@ -120,7 +117,7 @@ func PostBaseFileSaveFile(rootPath string, c *gin.Context) {
120117
}
121118

122119
func PostBaseFileSaveFiles(rootPath string, c *gin.Context) {
123-
fsSvc, err := getBaseFileFsSvc(rootPath)
120+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
124121
if err != nil {
125122
HandleErrorInternalServerError(c, err)
126123
return
@@ -181,7 +178,7 @@ func PostBaseFileSaveDir(rootPath string, c *gin.Context) {
181178
return
182179
}
183180

184-
fsSvc, err := getBaseFileFsSvc(rootPath)
181+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
185182
if err != nil {
186183
HandleErrorBadRequest(c, err)
187184
return
@@ -205,7 +202,7 @@ func PostBaseFileRenameFile(rootPath string, c *gin.Context) {
205202
return
206203
}
207204

208-
fsSvc, err := getBaseFileFsSvc(rootPath)
205+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
209206
if err != nil {
210207
HandleErrorBadRequest(c, err)
211208
return
@@ -229,7 +226,7 @@ func DeleteBaseFileFile(rootPath string, c *gin.Context) {
229226
payload.Path = "."
230227
}
231228

232-
fsSvc, err := getBaseFileFsSvc(rootPath)
229+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
233230
if err != nil {
234231
HandleErrorBadRequest(c, err)
235232
return
@@ -257,7 +254,7 @@ func PostBaseFileCopyFile(rootPath string, c *gin.Context) {
257254
return
258255
}
259256

260-
fsSvc, err := getBaseFileFsSvc(rootPath)
257+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
261258
if err != nil {
262259
HandleErrorBadRequest(c, err)
263260
return
@@ -272,7 +269,7 @@ func PostBaseFileCopyFile(rootPath string, c *gin.Context) {
272269
}
273270

274271
func PostBaseFileExport(rootPath string, c *gin.Context) {
275-
fsSvc, err := getBaseFileFsSvc(rootPath)
272+
fsSvc, err := fs.GetBaseFileFsSvc(rootPath)
276273
if err != nil {
277274
HandleErrorBadRequest(c, err)
278275
return
@@ -289,14 +286,3 @@ func PostBaseFileExport(rootPath string, c *gin.Context) {
289286
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath))
290287
c.File(zipFilePath)
291288
}
292-
293-
func GetBaseFileFsSvc(rootPath string) (svc interfaces.FsService, err error) {
294-
return getBaseFileFsSvc(rootPath)
295-
}
296-
297-
func getBaseFileFsSvc(rootPath string) (svc interfaces.FsService, err error) {
298-
workspacePath := utils.GetWorkspace()
299-
fsSvc := fs.NewFsService(filepath.Join(workspacePath, rootPath))
300-
301-
return fsSvc, nil
302-
}

‎core/controllers/spider.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"github.com/crawlab-team/crawlab/core/constants"
66
"github.com/crawlab-team/crawlab/core/models/models"
77
mongo2 "github.com/crawlab-team/crawlab/core/mongo"
8+
"github.com/crawlab-team/crawlab/core/spider"
89
"math"
910
"os"
1011
"path/filepath"
@@ -293,6 +294,17 @@ func PostSpider(c *gin.Context) {
293294
return
294295
}
295296

297+
// create template if available
298+
if utils.IsPro() && s.Template != "" {
299+
if templateSvc := spider.GetSpiderTemplateRegistryService(); templateSvc != nil {
300+
err = templateSvc.CreateTemplate(s.Id)
301+
if err != nil {
302+
HandleErrorInternalServerError(c, err)
303+
return
304+
}
305+
}
306+
}
307+
296308
HandleSuccessWithData(c, s)
297309
}
298310

‎core/fs/utils.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package fs
2+
3+
import (
4+
"github.com/crawlab-team/crawlab/core/interfaces"
5+
"github.com/crawlab-team/crawlab/core/utils"
6+
"path/filepath"
7+
)
8+
9+
func GetBaseFileFsSvc(rootPath string) (svc interfaces.FsService, err error) {
10+
workspacePath := utils.GetWorkspace()
11+
fsSvc := NewFsService(filepath.Join(workspacePath, rootPath))
12+
13+
return fsSvc, nil
14+
}

‎core/interfaces/spider_template_service.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package interfaces
2+
3+
import "go.mongodb.org/mongo-driver/bson/primitive"
4+
5+
type SpiderTemplateService interface {
6+
CreateTemplate(id primitive.ObjectID) (err error)
7+
}

‎core/models/models/spider.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ type Spider struct {
2020
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
2121
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
2222
Git *Git `json:"git,omitempty" bson:"-"`
23+
Template string `json:"template,omitempty" bson:"template,omitempty"` // spider template
24+
TemplateParams *struct {
25+
SpiderName string `json:"spider_name,omitempty" bson:"spider_name,omitempty"`
26+
StartUrls string `json:"start_urls,omitempty" bson:"start_urls,omitempty"`
27+
Domains string `json:"domains,omitempty" bson:"domains,omitempty"`
28+
} `json:"template_params,omitempty" bson:"template_params,omitempty"`
2329

2430
// stats
2531
Stat *SpiderStat `json:"stat,omitempty" bson:"-"`

‎core/spider/registry_service.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package spider
2+
3+
import "github.com/crawlab-team/crawlab/core/interfaces"
4+
5+
var templateSvcInstance interfaces.SpiderTemplateService
6+
7+
func SetSpiderTemplateRegistryService(svc interfaces.SpiderTemplateService) {
8+
templateSvcInstance = svc
9+
}
10+
11+
func GetSpiderTemplateRegistryService() interfaces.SpiderTemplateService {
12+
return templateSvcInstance
13+
}

‎core/task/handler/runner.go

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,19 @@ func (r *Runner) startHealthCheck() {
340340
}
341341
}
342342

343+
// configurePythonPath sets up the Python environment paths, handling both pyenv and default installations
344+
func (r *Runner) configurePythonPath() {
345+
// Configure global node_modules path
346+
pyenvRoot := utils.GetPyenvPath()
347+
pyenvShimsPath := pyenvRoot + "/shims"
348+
pyenvBinPath := pyenvRoot + "/bin"
349+
350+
// Configure global pyenv path
351+
_ = os.Setenv("PYENV_ROOT", pyenvRoot)
352+
_ = os.Setenv("PATH", pyenvShimsPath+":"+os.Getenv("PATH"))
353+
_ = os.Setenv("PATH", pyenvBinPath+":"+os.Getenv("PATH"))
354+
}
355+
343356
// configureNodePath sets up the Node.js environment paths, handling both nvm and default installations
344357
func (r *Runner) configureNodePath() {
345358
// Configure nvm-based Node.js paths
@@ -366,7 +379,10 @@ func (r *Runner) configureGoPath() {
366379
// - Crawlab-specific variables
367380
// - Global environment variables from the system
368381
func (r *Runner) configureEnv() {
369-
// Configure Node.js paths
382+
// Configure Python path
383+
r.configurePythonPath()
384+
385+
// Configure Node.js path
370386
r.configureNodePath()
371387

372388
// Configure Go path
@@ -375,8 +391,6 @@ func (r *Runner) configureEnv() {
375391
// Default envs
376392
r.cmd.Env = os.Environ()
377393
r.cmd.Env = append(r.cmd.Env, "CRAWLAB_TASK_ID="+r.tid.Hex())
378-
r.cmd.Env = append(r.cmd.Env, "PYENV_ROOT="+utils.PyenvRoot)
379-
r.cmd.Env = append(r.cmd.Env, "PATH="+os.Getenv("PATH")+":"+utils.PyenvRoot+"/shims:"+utils.PyenvRoot+"/bin")
380394

381395
// Global environment variables
382396
envs, err := client.NewModelService[models.Environment]().GetMany(nil, nil)

‎core/utils/config.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ const (
3030
DefaultInstallRoot = "/app/install"
3131
MetadataConfigDirName = ".crawlab"
3232
MetadataConfigName = "config.json"
33-
PyenvRoot = "/root/.pyenv"
33+
DefaultPyenvPath = "/root/.pyenv"
3434
DefaultNodeModulesPath = "/usr/lib/node_modules"
3535
DefaultGoPath = "/root/go"
3636
)
@@ -250,6 +250,13 @@ func GetInstallRoot() string {
250250
return DefaultInstallRoot
251251
}
252252

253+
func GetPyenvPath() string {
254+
if res := viper.GetString("install.pyenv.path"); res != "" {
255+
return res
256+
}
257+
return DefaultPyenvPath
258+
}
259+
253260
func GetNodeModulesPath() string {
254261
if res := viper.GetString("install.node.path"); res != "" {
255262
return res

‎core/utils/string.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package utils
2+
3+
import (
4+
"golang.org/x/text/cases"
5+
"golang.org/x/text/language"
6+
"strings"
7+
)
8+
9+
func ToSnakeCase(s string) string {
10+
s = strings.TrimSpace(s)
11+
s = strings.ToLower(s)
12+
s = strings.ReplaceAll(s, " ", "_")
13+
s = strings.ReplaceAll(s, "-", "_")
14+
return s
15+
}
16+
17+
func ToPascalCase(s string) string {
18+
s = strings.TrimSpace(s)
19+
s = strings.ReplaceAll(s, "_", " ")
20+
s = cases.Title(language.English).String(s)
21+
s = strings.ReplaceAll(s, " ", "")
22+
return s
23+
}

0 commit comments

Comments
 (0)