run through CI locally

README.md

@@ -1,8 +1,24 @@
-test pipeline
-
 basic structure from alibaba examples.
 
-each mvn profile updates named CloudFlow in Ali Cloud.
+each mvn project updates named FC of CloudFlow in Ali Cloud.
+
+testdata-pipeline.yaml: 测试用公共云函数
+
+- CloudFlow: https://fnf.console.aliyun.com/cn-beijing/flows/test-pipeline-dev/flow-edit
+
+prod-pipeline.yaml: 生产用公共云函数
+
+- todo
+
+具体算法 (嵌入到测试用CloudFlow和生产用CloudFlow)
+
+- image-dup-check.yaml: 图片查重云函数 image-dup-check-feature-extract 和 image-dup-check-result-generate
+- ...
 
-update.list defines which FC should be updated online
+执行
 
+- 配置阿里云AK，https://www.serverless-devs.com/docs/getting-started
+- `s -t testdata-pipeline.yaml preview`
+- `s -t testdata-pipeline.yaml verify`
+- `s -t testdata-pipeline.yaml deploy`
+- `s cli fc3 invoke --region cn-beijing --function-name ReadOssTestDataHandler -e '{"ossPrefix": "/mnt/yyc-algo-test/image-dup-check/multi-modal-embedding-img-search/test/positive/"}'`

publish.py

@@ -1,143 +0,0 @@
-import pwd
-import subprocess
-import time
-from pathlib import Path
-import zipfile
-import os
-import oss2
-import shutil
-
-
-def path_is_parent(parent_path, child_path):
-    parent_path = os.path.abspath(parent_path)
-    child_path = os.path.abspath(child_path)
-    return os.path.commonpath([parent_path]) == os.path.commonpath([parent_path, child_path])
-
-
-def is_ignored(target_path, ignores):
-    for ignore in ignores:
-        if os.path.relpath(target_path, ignore) == '.' or path_is_parent(ignore, target_path):
-            return True
-    return False
-
-
-def zip_file(workspace, eve_app):
-    os.chdir('%s/%s/src' % (workspace, eve_app))
-    if os.path.isfile('README.md'):
-        shutil.copy('README.md', 'code')
-    elif os.path.isfile('readme.md'):
-        shutil.copy('readme.md', 'code')
-    elif os.path.isfile('Readme.md'):
-        shutil.copy('Readme.md', 'code')
-    os.chdir('%s/%s/src/code' % (workspace, eve_app))
-    ignore_list = ['./.git', './.github',
-                   './.idea', './.DS_Store', './.vscode']
-    with zipfile.ZipFile('code.zip', mode="w") as f:
-        for dirpath, dirnames, filenames in os.walk('./'):
-            if dirpath != './' and is_ignored(dirpath, ignore_list):
-                continue
-            for filename in filenames:
-                absoult_file_path = os.path.join(dirpath, filename)
-                if not is_ignored(absoult_file_path, ignore_list) and "code.zip" not in filename:
-                    f.write(os.path.join(dirpath, filename))
-
-    return 'code.zip'
-
-
-def zip_golang_binary(workspace, eve_app):
-    os.chdir('%s/%s/src' % (workspace, eve_app))
-    if os.path.isfile('README.md'):
-        shutil.copy('README.md', 'code/target')
-    elif os.path.isfile('readme.md'):
-        shutil.copy('readme.md', 'code/target')
-    elif os.path.isfile('Readme.md'):
-        shutil.copy('Readme.md', 'code/target')
-    os.chdir('%s/%s/src/code/target' % (workspace, eve_app))
-    ignore_list = ['./.git', './.github',
-                   './.idea', './.DS_Store', './.vscode']
-    with zipfile.ZipFile('code.zip', mode="w") as f:
-        for dirpath, dirnames, filenames in os.walk('./'):
-            if dirpath != './' and is_ignored(dirpath, ignore_list):
-                continue
-            for filename in filenames:
-                absoult_file_path = os.path.join(dirpath, filename)
-                if not is_ignored(absoult_file_path, ignore_list) and "code.zip" not in filename:
-                    f.write(os.path.join(dirpath, filename))
-
-    return 'code.zip'
-
-
-def upload_oss(code_name, zip_file):
-    auth = oss2.Auth(os.environ.get('AccessKeyId'),
-                     os.environ.get('AccessKeySecret'))
-    bucket = oss2.Bucket(auth, os.environ.get(
-        'ArtifactEndpoint'), os.environ.get('ArtifactBucket'))
-
-    with open(zip_file, 'rb') as fileobj:
-        object_name = '%s/code.zip' % (code_name)
-        bucket.put_object(object_name, fileobj)
-
-
-workspace = os.getcwd()
-with open('update.list') as f:
-    publish_list = [eve_app.strip() for eve_app in f.readlines()]
-
-failed_registry = []
-failed_oss = []
-for eve_app in publish_list:
-    times = 1
-    os.chdir(workspace)
-    try:
-        while times <= 3:
-            print("----------------------: ", eve_app)
-            # publish app to registry
-            publish_script = 'https://serverless-registry.oss-cn-hangzhou.aliyuncs.com/publish-file/python3/hub-publish.py'
-            command = 'cd %s && wget %s && python hub-publish.py' % (
-                eve_app, publish_script)
-            child = subprocess.Popen(
-                command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, )
-            stdout, stderr = child.communicate()
-            if child.returncode == 0:
-                print("stdout:", stdout.decode("utf-8"))
-                break
-            else:
-                print("stdout:", stdout.decode("utf-8"))
-                print("stderr:", stderr.decode("utf-8"))
-                time.sleep(3)
-                if times == 3:
-                    raise ChildProcessError(stderr)
-            times = times + 1
-    except Exception as e:
-        print('Failed to publish registry, app %s, err: %s' % (eve_app, e))
-        failed_registry.append(eve_app)
-
-    # publish code.zip to oss
-    os.chdir(workspace)
-    try:
-        makefile = Path('%s/src/Makefile' % (eve_app))
-        if makefile.is_file():
-            print("----------------------Makefile: ", makefile)
-            command = 'cd %s/src && make release' % (eve_app)
-            print(command)
-            child = subprocess.Popen(
-                command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, )
-            stdout, stderr = child.communicate()
-            print("stdout:", stdout.decode("utf-8"))
-            if child.returncode != 0:
-                print("stderr:", stderr.decode("utf-8"))
-                raise ChildProcessError(stderr)
-        jarPath = '%s/%s/src/code/target/code.jar' % (workspace, eve_app)
-        golangBinaryPath = '%s/%s/src/code/target/main' % (workspace, eve_app)
-        if os.path.isfile(jarPath):
-            code_zip = jarPath
-        elif os.path.isfile(golangBinaryPath):
-            code_zip = zip_golang_binary(workspace, eve_app)
-        else:
-            code_zip = zip_file(workspace, eve_app)
-        upload_oss(eve_app, code_zip)
-    except Exception as e:
-        print('Failed to publish oss, app %s, err: %s' % (eve_app, e))
-        failed_oss.append(eve_app)
-
-print('Failed registry list: ', failed_registry)
-print('Failed oss list: ', failed_oss)

testdata-pipeline.yaml

@@ -14,7 +14,7 @@
 #   更多函数计算案例，可参考：https://github.com/devsapp/awesome/
 #   有问题快来钉钉群问一下吧：33947367
 # ------------------------------------
-edition: 1.0.0
+edition: 3.0.0
 name: testdata-pipeline
 # access 是当前应用所需要的密钥信息配置：
 # 密钥配置可以参考：https://www.serverless-devs.com/serverless-devs/command/config
@@ -28,15 +28,15 @@ vars: # 全局变量
     instance: dev
     description: ""
 
-services:
+resources:
   read-oss-test-data: # 业务名称/模块名称
     # 如果只想针对 read-oss-test-data 下面的业务进行相关操作，可以在命令行中加上 read-oss-test-data，例如：
     # 只对read-oss-test-data进行构建：s read-oss-test-data build
     # 如果不带有 read-oss-test-data ，而是直接执行 s build，工具则会对当前Yaml下，所有和 read-oss-test-data 平级的业务模块（如有其他平级的模块，例如下面注释的next-function），按照一定顺序进行 build 操作
-    component: fc # 组件名称，Serverless Devs 工具本身类似于一种游戏机，不具备具体的业务能力，组件类似于游戏卡，用户通过向游戏机中插入不同的游戏卡实现不同的功能，即通过使用不同的组件实现不同的具体业务能力
+    component: fc3 # 组件名称，Serverless Devs 工具本身类似于一种游戏机，不具备具体的业务能力，组件类似于游戏卡，用户通过向游戏机中插入不同的游戏卡实现不同的功能，即通过使用不同的组件实现不同的具体业务能力
     actions: # 自定义执行逻辑，关于actions 的使用，可以参考：https://www.serverless-devs.com/serverless-devs/yaml#行为描述
       pre-deploy: # 在deploy之前运行
-        - run: mvn package
+        - run: mvn clean package -pl read-oss-test-data
           path: ./
     #        - component: fc build --use-docker --dockerfile ./code/Dockerfile  # 要运行的组件，格式为【component: 组件名 命令 参数】（可以通过s cli registry search --type Component 获取组件列表）
     #        - run: docker build xxx          # 要执行的系统命令，类似于一种钩子的形式
@@ -48,14 +48,39 @@ services:
     #        - component: fc versions publish # 要运行的命令行
     props:
       region: ${vars.region} # 关于变量的使用方法，可以参考：https://www.serverless-devs.com/serverless-devs/yaml#变量赋值
-      service: ${vars.service}
-      function:
-        name: read-oss-test-data
-        description: ""
-        runtime: java11
-        codeUri: ./target
-        handler: com.yaoyicloud.ai.pipeline.fc.ReadOssTestDataHandler::handleRequest
-        memorySize: 128
-        timeout: 300
-        # initializationTimeout: 60
-        # initializer: example.App::initialize
+      service:
+        name: ${vars.service.name}
+        description: ${vars.service.description}
+      ossMountConfig:
+        mountPoints:
+          - endpoint: http://oss-cn-beijing.aliyuncs.com
+            bucketName: yyc-algo-test
+            mountDir: /mnt/yyc-algo-test
+            readOnly: true
+      functionName: ReadOssTestDataHandler
+      runtime: java11
+      code: ./read-oss-test-data/target/read-oss-test-data-1.0-SNAPSHOT.jar
+      handler: com.yaoyicloud.ai.pipeline.fc.ReadOssTestDataHandler::handleRequest
+      role: acs:ram::1861991840278303:role/fc-oss-read
+      memorySize: 512
+      timeout: 300
+      # initializationTimeout: 60
+      # initializer: example.App::initialize
+
+  mimic-real-msg: # 业务名称/模块名称
+    component: fc3
+    actions:
+      pre-deploy:
+        - run: mvn clean package -pl mimic-real-msg
+          path: ./
+    props:
+      region: ${vars.region}
+      service:
+        name: ${vars.service.name}
+        description: ${vars.service.description}
+      functionName: MimicRealMsg
+      runtime: java11
+      code: ./mimic-real-msg/target/mimic-real-msg-1.0-SNAPSHOT.jar
+      handler: com.yaoyicloud.ai.pipeline.fc.MimicRealMsgHandler::handleRequest
+      memorySize: 512
+      timeout: 300

update.list

@@ -1,2 +0,0 @@
-quick-start-sample-codes/quick-start-sample-codes-golang/rabbit-trigger-golang
-quick-start-sample-codes/quick-start-sample-codes-python/rabbitmq-trigger-python