From ca820733d9cc918e04891e49df15115c1066ae48 Mon Sep 17 00:00:00 2001
From: Brian Wandell <wandell@stanford.edu>
Date: Mon, 23 Oct 2017 10:05:23 -0700
Subject: [PATCH] Rearranging for ultimate separation of cloud control. Added
functionality for Alicloud.
---
Alicloud/aclRootPath.m | 17 ++++++++
acloud/acloud.m | 91 ++++++++++++++++++++++++++++++++----------
gcloud/gcloud.m | 55 -------------------------
gcloud/kubernetes.m | 40 -------------------
4 files changed, 86 insertions(+), 117 deletions(-)
create mode 100644 Alicloud/aclRootPath.m
delete mode 100644 gcloud/gcloud.m
delete mode 100644 gcloud/kubernetes.m
diff --git a/Alicloud/aclRootPath.m b/Alicloud/aclRootPath.m
new file mode 100644
index 0000000..1d8b61e
--- /dev/null
+++ b/Alicloud/aclRootPath.m
@@ -0,0 +1,17 @@
+function rootPath=aclRootPath()
+%%isetbioRootPath Return the path to the root isetbio directory
+%
+% Syntax:
+% rootPath=aclRootPath;
+%
+% Description:
+% This points at the top level of the acloud tree on the Matlab path.
+%
+
+%% Get path to this function and then walk back up to the isetbio root.
+pathToMe = mfilename('fullpath');
+
+%% Walk back up the chain
+rootPath = fileparts(pathToMe);
+
+end
diff --git a/acloud/acloud.m b/acloud/acloud.m
index 3b35e3f..f7225b0 100644
--- a/acloud/acloud.m
+++ b/acloud/acloud.m
@@ -6,18 +6,41 @@ classdef acloud < handle
%
% ZL Vistasoft Team 2017
properties
- bucket = 'oss://';
- ros = 'python /Library/Frameworks/Python.framework/Versions/2.7/bin/ros --json';
+ bucket;
+ ros; % 'python /Library/Frameworks/Python.framework/Versions/2.7/bin/ros --json';
+ kubeTemplate; % Template for the kubernetes cluster
+ python; % Many people have two versions of python. This is the executable you want used for alicloud.
end
methods
function obj = acloud(varargin)
-% p = inputParser;
-% p.addParameter('bucket','oss://',@ischar);
-% p.addParameter('ros','python /Library/Frameworks/Python.framework/Versions/2.7/bin/ros --json',@ischar)
-% p.parse(varargin{:});
-% obj.bucket = p.Results.bucket;
-% obj.ros = p.Results.ros;
+ % Users may vary in the version of python, ossutil, ros, and template
+ % locations. So we let them be set here.
+ %
+ % acloud('python','/Users/wandell/anaconda/bin/python')
+ %
+ p = inputParser;
+ p.addParameter('bucket','oss://',@ischar);
+
+ [~,systemPython] = system('which python');
+ p.addParameter('python',systemPython,@(x)(exist(x,'file')));
+
+ kubeTemplate = fullfile(aclRootPath,'kube_3master.json');
+ p.addParameter('kubeTemplate',kubeTemplate,@(x)(exist(x,'file')));
+
+ p.parse(varargin{:});
+
+ % ROS is in the python bin directory. We force the output
+ % to be JSON.
+ obj.bucket = p.Results.bucket;
+ obj.kubeTemplate = p.Results.kubeTemplate;
+ obj.python = p.Results.python;
+
+ pythonDir = fileparts(obj.python);
+ obj.ros = fullfile(pythonDir,'ros --json');
+
end
+
+ % List the objects
function [result, status, cmd] = ls(obj,bucketname)
if ieNotDefined('bucketname')
d = obj.bucket;
@@ -27,6 +50,8 @@ classdef acloud < handle
cmd = sprintf('ossutil ls %s\n',d);
[status,result] = system(cmd);
end
+
+ % Remove one object
function [result, status, cmd] = objectrm(obj,objectname)
if ieNotDefined('objectname')
disp('Object name required')
@@ -36,6 +61,8 @@ classdef acloud < handle
[status, result] = system(cmd);
end
end
+
+ % Remove one bucket
function [result, status, cmd] = bucketrm(obj,bucketname)
if ieNotDefined('bucketname')
disp('Bucket name required')
@@ -45,6 +72,8 @@ classdef acloud < handle
[status, result] = system(cmd);
end
end
+
+ % Create one bucket
function [result, status, cmd] = bucketCreate(obj,bucketname)
if ieNotDefined('bucketname')
disp('Bucket name (lower case) required')
@@ -55,48 +84,66 @@ classdef acloud < handle
[status, result] = system(cmd);
end
end
+
+ % Upload a folder or a file
function [result, status, cmd] = upload(obj,local_dir,cloud_dir)
cloud_dir = fullfile(obj.bucket,cloud_dir);
cmd = sprintf('ossutil cp %s %s -r -f -u\n',local_dir,cloud_dir);
[status, result] = system(cmd);
end
+
+ % Download a folder or a file
function [result, status, cmd] = download(obj,cloud_dir,local_dir)
cloud_dir = fullfile(obj.bucket,cloud_dir);
cmd = sprintf('ossutil cp %s %s -r -f -u\n',cloud_dir,local_dir);
[status, result] = system(cmd);
end
- function [result, status,masterIp, stackname ,stackID, cmd] = k8sCreate(obj,stackname, MasterInstanceType,WorkerInstanceType,NumberOfNodes,password)
+
+ % Create a kubernetes cluster that can be controled by kubectl
+ function [result, status,masterIp, stackname ,stackID, cmd] = ...
+ k8sCreate(obj,stackname, MasterInstanceType,WorkerInstanceType,NumberOfNodes,password)
+
cmd = sprintf('%s create-stack --stack-name %s --template-url /Users/eugeneliu/git_repo/RenderToolbox4/Alicloud/kube_3master.json --parameters MasterInstanceType=%s,WorkerInstanceType=%s,ImageId=centos_7,NumOfNodes=%d,LoginPassword=%s',...
obj.ros,stackname,MasterInstanceType,WorkerInstanceType,NumberOfNodes,password);
+
[~, result] = system(cmd);
result = erase(result,'[Succeed]');
result = parse_json(result);
stackID = result.Id;
- while 1
- cmd = sprintf('%s describe-stack --stack-name %s --stack-id %s',obj.ros,stackname,stackID);
- [~, result] = system(cmd);
- result_check = erase(result,'[Succeed]');
- result_check = parse_json(result_check);
- status = result_check.Status;
- pause(60);
- fprintf('%s\n',status);
- if strcmp(status,'CREATE_COMPLETE')== 1
- break;
- end
- masterIp = result_check.Outputs{2}.OutputValue;
- end
+ while true
+ % Loop to check the status of the creation. Return
+ % when done.
+ cmd = sprintf('%s describe-stack --stack-name %s --stack-id %s',obj.ros,stackname,stackID);
+ [~, result] = system(cmd);
+ result_check = erase(result,'[Succeed]');
+ result_check = parse_json(result_check);
+ status = result_check.Status;
+
+ % Check again in 60 secs
+ pause(60);
+ fprintf('%s\n',status);
+ if strcmp(status,'CREATE_COMPLETE')== 1
+ break;
+ end
+ masterIp = result_check.Outputs{2}.OutputValue;
+ end
end
+
+ % Delete a kubernetes cluster
function[result, status, cmd]= k8sDelete(obj,stackname,stackID)
cmd = sprintf('%s delete-stack --region-id us-west-1 --stack-name %s --stack-id %s',obj.ros,stackname,stackID);
[status, result] = system(cmd);
end
+
+ % Adjust the kubernetes allocation
function[result, status, cmd]=k8sUpdate(obj,stackname,stackID,MasterInstanceType,WorkerInstanceType,NumberOfNodes,password)
cmd = sprintf('%s create-stack --stack-name %s --template-url /Users/eugeneliu/git_repo/RenderToolbox4/Alicloud/kube_3master.json --parameters MasterInstanceType=%s,WorkerInstanceType=%s,ImageId=centos_7,NumOfNodes=%d,LoginPassword=%s,--regionID us-west-1 --stack-id %s',...
obj.ros,stackname,MasterInstanceType,WorkerInstanceType,NumberOfNodes,password,stackID);
[status, result] = system(cmd);
end
+
end
end
diff --git a/gcloud/gcloud.m b/gcloud/gcloud.m
deleted file mode 100644
index 647fe83..0000000
--- a/gcloud/gcloud.m
+++ /dev/null
@@ -1,55 +0,0 @@
-classdef gcloud < handle
- %
- % gcloud(command,varargin)
- %
- % ls: @gcloud('ls','directory')
- % rm: @gcloud('rm','fullpath to file')
- %e
-
- properties
- bucket = '';
- end
-
- methods
- function obj = gcloud(varargin)
- p = inputParser;
- p.addParameter('bucket','gs://primal-surfer-140120.appspot.com/',@ischar);
-
- p.parse(varargin{:});
-
- obj.bucket = p.Results.bucket;
- end
-
- function [result, status] = ls(obj,directory)
- if ieNotDefined('directory');
- d = obj.bucket;
- else
- d = fullfile(obj.bucket, directory);
- end
-
- cmd = sprintf('gsutil ls %s\n',d);
- [status,result] = system(cmd);
- result1 = textscan(result,'%s','Delimiter','\n','CollectOutput',true);
- result = result1{1};
- if strncmp(result{1},'CommandException',7)
- disp(result{1})
- return;
- else
- nFiles = size(result,1);
- nSkip = length(obj.bucket);
- for ii=1:nFiles
- result{ii} = result{ii}((nSkip+1):end);
- end
- end
-
- end
-
- function [result,status] = rm(obj,fname)
- %
- filename = fullfile(obj.bucket, fname);
- cmd = sprintf('gsutil rm %s\n',filename);
- [status,result] = system(cmd);
- end
-
- end
-end
diff --git a/gcloud/kubernetes.m b/gcloud/kubernetes.m
deleted file mode 100644
index 38bb835..0000000
--- a/gcloud/kubernetes.m
+++ /dev/null
@@ -1,40 +0,0 @@
-classdef kubernetes < handle
- % Interface to remember different kubernetes (kubectl) commands
- %
- properties
-
- % User's name space. Set when the object is created.
- namespace = '';
-
- end
-
- methods
- function obj = kubernetes(namespace,varargin)
- p = inputParser;
- p.addRequired('namespace',@ischar);
- p.parse(namespace,varargin{:});
-
- obj.namespace = p.Results.namespace;
-
- end
-
- % List the jobs running in your name space
- function [result,status] = jobs(obj,varargin)
-
- p = inputParser;
- p.addParameter('recipeName','*',@ischar);
- p.parse(varargin{:});
- recipeName = p.Results.recipeName;
- % Could also add 'recipeName' and grep on that
- if recipeName == '*'
- cmd = sprintf('kubectl get jobs --namespace=%s',obj.namespace);
- else
- cmd = sprintf('kubectl get jobs --namespace=%s | grep %s',obj.namespace,recipeName);
- end
-
- % cmd = sprintf('kubectl get jobs --namespace=%s | grep %s',obj.namespace,recipeName);
- [status, result] = system(cmd);
- end
-
- end
-end
--
GitLab