Skip to content
Snippets Groups Projects
Commit ca820733 authored by Brian Wandell's avatar Brian Wandell
Browse files

Rearranging for ultimate separation of cloud control. Added functionality for Alicloud.

parent b8f00132
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Alicloud/aclRootPath.m 0 → 100644
+ 17
− 0
View file @ ca820733
function rootPath=aclRootPath()
%%isetbioRootPath Return the path to the root isetbio directory
%
% Syntax:
% rootPath=aclRootPath;
%
% Description:
% This points at the top level of the acloud tree on the Matlab path.
%
%% Get path to this function and then walk back up to the isetbio root.
pathToMe = mfilename('fullpath');
%% Walk back up the chain
rootPath = fileparts(pathToMe);
end
acloud/acloud.m
+ 69
− 22
View file @ ca820733
......@@ -6,18 +6,41 @@ classdef acloud < handle
%
% ZL Vistasoft Team 2017
properties
bucket = 'oss://';
ros = 'python /Library/Frameworks/Python.framework/Versions/2.7/bin/ros --json';
bucket;
ros; % 'python /Library/Frameworks/Python.framework/Versions/2.7/bin/ros --json';
kubeTemplate; % Template for the kubernetes cluster
python; % Many people have two versions of python. This is the executable you want used for alicloud.
end
methods
function obj = acloud(varargin)
% p = inputParser;
% p.addParameter('bucket','oss://',@ischar);
% p.addParameter('ros','python /Library/Frameworks/Python.framework/Versions/2.7/bin/ros --json',@ischar)
% p.parse(varargin{:});
% obj.bucket = p.Results.bucket;
% obj.ros = p.Results.ros;
% Users may vary in the version of python, ossutil, ros, and template
% locations. So we let them be set here.
%
% acloud('python','/Users/wandell/anaconda/bin/python')
%
p = inputParser;
p.addParameter('bucket','oss://',@ischar);
[~,systemPython] = system('which python');
p.addParameter('python',systemPython,@(x)(exist(x,'file')));
kubeTemplate = fullfile(aclRootPath,'kube_3master.json');
p.addParameter('kubeTemplate',kubeTemplate,@(x)(exist(x,'file')));
p.parse(varargin{:});
% ROS is in the python bin directory. We force the output
% to be JSON.
obj.bucket = p.Results.bucket;
obj.kubeTemplate = p.Results.kubeTemplate;
obj.python = p.Results.python;
pythonDir = fileparts(obj.python);
obj.ros = fullfile(pythonDir,'ros --json');
end
% List the objects
function [result, status, cmd] = ls(obj,bucketname)
if ieNotDefined('bucketname')
d = obj.bucket;
......@@ -27,6 +50,8 @@ classdef acloud < handle
cmd = sprintf('ossutil ls %s\n',d);
[status,result] = system(cmd);
end
% Remove one object
function [result, status, cmd] = objectrm(obj,objectname)
if ieNotDefined('objectname')
disp('Object name required')
......@@ -36,6 +61,8 @@ classdef acloud < handle
[status, result] = system(cmd);
end
end
% Remove one bucket
function [result, status, cmd] = bucketrm(obj,bucketname)
if ieNotDefined('bucketname')
disp('Bucket name required')
......@@ -45,6 +72,8 @@ classdef acloud < handle
[status, result] = system(cmd);
end
end
% Create one bucket
function [result, status, cmd] = bucketCreate(obj,bucketname)
if ieNotDefined('bucketname')
disp('Bucket name (lower case) required')
......@@ -55,48 +84,66 @@ classdef acloud < handle
[status, result] = system(cmd);
end
end
% Upload a folder or a file
function [result, status, cmd] = upload(obj,local_dir,cloud_dir)
cloud_dir = fullfile(obj.bucket,cloud_dir);
cmd = sprintf('ossutil cp %s %s -r -f -u\n',local_dir,cloud_dir);
[status, result] = system(cmd);
end
% Download a folder or a file
function [result, status, cmd] = download(obj,cloud_dir,local_dir)
cloud_dir = fullfile(obj.bucket,cloud_dir);
cmd = sprintf('ossutil cp %s %s -r -f -u\n',cloud_dir,local_dir);
[status, result] = system(cmd);
end
function [result, status,masterIp, stackname ,stackID, cmd] = k8sCreate(obj,stackname, MasterInstanceType,WorkerInstanceType,NumberOfNodes,password)
% Create a kubernetes cluster that can be controled by kubectl
function [result, status,masterIp, stackname ,stackID, cmd] = ...
k8sCreate(obj,stackname, MasterInstanceType,WorkerInstanceType,NumberOfNodes,password)
cmd = sprintf('%s create-stack --stack-name %s --template-url /Users/eugeneliu/git_repo/RenderToolbox4/Alicloud/kube_3master.json --parameters MasterInstanceType=%s,WorkerInstanceType=%s,ImageId=centos_7,NumOfNodes=%d,LoginPassword=%s',...
obj.ros,stackname,MasterInstanceType,WorkerInstanceType,NumberOfNodes,password);
[~, result] = system(cmd);
result = erase(result,'[Succeed]');
result = parse_json(result);
stackID = result.Id;
while 1
cmd = sprintf('%s describe-stack --stack-name %s --stack-id %s',obj.ros,stackname,stackID);
[~, result] = system(cmd);
result_check = erase(result,'[Succeed]');
result_check = parse_json(result_check);
status = result_check.Status;
pause(60);
fprintf('%s\n',status);
if strcmp(status,'CREATE_COMPLETE')== 1
break;
end
masterIp = result_check.Outputs{2}.OutputValue;
end
while true
% Loop to check the status of the creation. Return
% when done.
cmd = sprintf('%s describe-stack --stack-name %s --stack-id %s',obj.ros,stackname,stackID);
[~, result] = system(cmd);
result_check = erase(result,'[Succeed]');
result_check = parse_json(result_check);
status = result_check.Status;
% Check again in 60 secs
pause(60);
fprintf('%s\n',status);
if strcmp(status,'CREATE_COMPLETE')== 1
break;
end
masterIp = result_check.Outputs{2}.OutputValue;
end
end
% Delete a kubernetes cluster
function[result, status, cmd]= k8sDelete(obj,stackname,stackID)
cmd = sprintf('%s delete-stack --region-id us-west-1 --stack-name %s --stack-id %s',obj.ros,stackname,stackID);
[status, result] = system(cmd);
end
% Adjust the kubernetes allocation
function[result, status, cmd]=k8sUpdate(obj,stackname,stackID,MasterInstanceType,WorkerInstanceType,NumberOfNodes,password)
cmd = sprintf('%s create-stack --stack-name %s --template-url /Users/eugeneliu/git_repo/RenderToolbox4/Alicloud/kube_3master.json --parameters MasterInstanceType=%s,WorkerInstanceType=%s,ImageId=centos_7,NumOfNodes=%d,LoginPassword=%s,--regionID us-west-1 --stack-id %s',...
obj.ros,stackname,MasterInstanceType,WorkerInstanceType,NumberOfNodes,password,stackID);
[status, result] = system(cmd);
end
end
end
......
gcloud/gcloud.m deleted 100644 → 0
+ 0
− 55
View file @ b8f00132
classdef gcloud < handle
%
% gcloud(command,varargin)
%
% ls: @gcloud('ls','directory')
% rm: @gcloud('rm','fullpath to file')
%e
properties
bucket = '';
end
methods
function obj = gcloud(varargin)
p = inputParser;
p.addParameter('bucket','gs://primal-surfer-140120.appspot.com/',@ischar);
p.parse(varargin{:});
obj.bucket = p.Results.bucket;
end
function [result, status] = ls(obj,directory)
if ieNotDefined('directory');
d = obj.bucket;
else
d = fullfile(obj.bucket, directory);
end
cmd = sprintf('gsutil ls %s\n',d);
[status,result] = system(cmd);
result1 = textscan(result,'%s','Delimiter','\n','CollectOutput',true);
result = result1{1};
if strncmp(result{1},'CommandException',7)
disp(result{1})
return;
else
nFiles = size(result,1);
nSkip = length(obj.bucket);
for ii=1:nFiles
result{ii} = result{ii}((nSkip+1):end);
end
end
end
function [result,status] = rm(obj,fname)
%
filename = fullfile(obj.bucket, fname);
cmd = sprintf('gsutil rm %s\n',filename);
[status,result] = system(cmd);
end
end
end
gcloud/kubernetes.m deleted 100644 → 0
+ 0
− 40
View file @ b8f00132
classdef kubernetes < handle
% Interface to remember different kubernetes (kubectl) commands
%
properties
% User's name space. Set when the object is created.
namespace = '';
end
methods
function obj = kubernetes(namespace,varargin)
p = inputParser;
p.addRequired('namespace',@ischar);
p.parse(namespace,varargin{:});
obj.namespace = p.Results.namespace;
end
% List the jobs running in your name space
function [result,status] = jobs(obj,varargin)
p = inputParser;
p.addParameter('recipeName','*',@ischar);
p.parse(varargin{:});
recipeName = p.Results.recipeName;
% Could also add 'recipeName' and grep on that
if recipeName == '*'
cmd = sprintf('kubectl get jobs --namespace=%s',obj.namespace);
else
cmd = sprintf('kubectl get jobs --namespace=%s | grep %s',obj.namespace,recipeName);
end
% cmd = sprintf('kubectl get jobs --namespace=%s | grep %s',obj.namespace,recipeName);
[status, result] = system(cmd);
end
end
end
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment