taskvine.h

Go to the documentation of this file.

/*
Copyright (C) 2022- The University of Notre Dame
This software is distributed under the GNU General Public License.
See the file COPYING for details.
*/
 
#ifndef TASKVINE_H
#define TASKVINE_H
 
#include "category.h"
#include "rmsummary.h"
#include "timestamp.h"
#include <sys/types.h>
 
struct vine_manager;
struct vine_task;
struct vine_file;
 
#define VINE_DEFAULT_PORT 9123 
#define VINE_RANDOM_PORT 0     
#define VINE_WAIT_FOREVER -1   
typedef enum {
        VINE_TRANSFER_ALWAYS = 0, 
        VINE_FIXED_LOCATION = 1, 
        VINE_WATCH = 2,          
        VINE_FAILURE_ONLY = 4,   
        VINE_SUCCESS_ONLY = 8,   
        VINE_RETRACT_ON_RESET = 16, 
        VINE_MOUNT_SYMLINK = 32,    
        VINE_MOUNT_MKDIR = 64       
} vine_mount_flags_t;
 
typedef enum {
        VINE_CACHE_LEVEL_TASK = 0,     
        VINE_CACHE_LEVEL_WORKFLOW = 1, 
        VINE_CACHE_LEVEL_WORKER = 2,   
        VINE_CACHE_LEVEL_FOREVER = 3   
} vine_cache_level_t;
 
typedef enum {
        VINE_PEER_NOSHARE = 1,    
        VINE_UNLINK_WHEN_DONE = 2 
} vine_file_flags_t;
 
typedef enum {
        VINE_SCHEDULE_UNSET = 0, 
        VINE_SCHEDULE_FCFS,      
        VINE_SCHEDULE_FILES,     
        VINE_SCHEDULE_TIME,      
        VINE_SCHEDULE_RAND,      
        VINE_SCHEDULE_WORST,     
        VINE_SCHEDULE_DISK       
} vine_schedule_t;
 
typedef enum {
        VINE_RESULT_SUCCESS = 0,        
        VINE_RESULT_INPUT_MISSING = 1,  
        VINE_RESULT_OUTPUT_MISSING = 2, 
        VINE_RESULT_STDOUT_MISSING = 4, 
        VINE_RESULT_SIGNAL = 1 << 3,    
        VINE_RESULT_RESOURCE_EXHAUSTION = 2 << 3, 
        VINE_RESULT_MAX_END_TIME = 3 << 3,  
        VINE_RESULT_UNKNOWN = 4 << 3,       
        VINE_RESULT_FORSAKEN = 5 << 3,      
        VINE_RESULT_MAX_RETRIES = 6 << 3,   
        VINE_RESULT_MAX_WALL_TIME = 7 << 3, 
        VINE_RESULT_RMONITOR_ERROR =
                        8 << 3, 
        VINE_RESULT_OUTPUT_TRANSFER_ERROR = 9 << 3,   
        VINE_RESULT_FIXED_LOCATION_MISSING = 10 << 3, 
        VINE_RESULT_CANCELLED = 11 << 3,              
        VINE_RESULT_LIBRARY_EXIT = 12 << 3,           
        VINE_RESULT_SANDBOX_EXHAUSTION = 13 << 3,     
        VINE_RESULT_MISSING_LIBRARY = 14 << 3         
} vine_result_t;
 
typedef enum {
        VINE_ALLOCATION_MODE_FIXED = CATEGORY_ALLOCATION_MODE_FIXED,
 
        VINE_ALLOCATION_MODE_MAX = CATEGORY_ALLOCATION_MODE_MAX,
 
        VINE_ALLOCATION_MODE_MIN_WASTE = CATEGORY_ALLOCATION_MODE_MIN_WASTE,
 
        VINE_ALLOCATION_MODE_MAX_THROUGHPUT = CATEGORY_ALLOCATION_MODE_MAX_THROUGHPUT,
 
        VINE_ALLOCATION_MODE_GREEDY_BUCKETING = CATEGORY_ALLOCATION_MODE_GREEDY_BUCKETING,
 
        VINE_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING = CATEGORY_ALLOCATION_MODE_EXHAUSTIVE_BUCKETING
} vine_category_mode_t;
 
typedef enum {
        VINE_FILE = 1,              
        VINE_URL,                   
        VINE_TEMP,                  
        VINE_BUFFER,                
        VINE_MINI_TASK,             
} vine_file_type_t;
 
 
struct vine_stats {
        /* Stats for the current state of workers: */
        int workers_connected; 
        int workers_init;      
        int workers_idle;      
        int workers_busy;      
        int workers_able;      
        /* Cumulative stats for workers: */
        int workers_joined;    
        int workers_removed;   
        int workers_released;  
        int workers_idled_out; 
        int workers_slow;      
        int workers_blocked;   
        int workers_lost;      
        /* Stats for the current state of tasks: */
        int tasks_waiting;      
        int tasks_on_workers;   
        int tasks_running;      
        int tasks_with_results; 
        int recovery_tasks_submitted; 
        /* Cumulative stats for tasks: */
        int tasks_submitted;  
        int tasks_dispatched; 
        int tasks_done;       
        int tasks_failed;     
        int tasks_cancelled;  
        int tasks_exhausted_attempts; 
        /* All times in microseconds */
        /* A time_when_* refers to an instant in time, otherwise it refers to a length of time. */
 
        /* Master time statistics: */
        timestamp_t time_when_started; 
        timestamp_t time_send; 
        timestamp_t time_receive;      
        timestamp_t time_send_good;    
        timestamp_t time_receive_good; 
        timestamp_t time_status_msgs;  
        timestamp_t time_internal;     
        timestamp_t time_polling; 
        timestamp_t time_application; 
        timestamp_t time_scheduling;  
        /* Workers time statistics: */
        timestamp_t time_workers_execute;            
        timestamp_t time_workers_execute_good;       
        timestamp_t time_workers_execute_exhaustion; 
        /* BW statistics */
        int64_t bytes_sent; 
        int64_t bytes_received; 
        double bandwidth; 
        /* resources statistics */
        int capacity_tasks;  
        int capacity_cores;  
        int capacity_memory; 
        int capacity_disk; 
        int capacity_gpus; 
        int capacity_instantaneous; 
        int capacity_weighted; 
        int64_t total_cores;  
        int64_t total_memory; 
        int64_t total_disk;   
        int64_t total_gpus;   
        int64_t committed_cores;  
        int64_t committed_memory; 
        int64_t committed_disk;   
        int64_t committed_gpus;   
        int64_t max_cores;  
        int64_t max_memory; 
        int64_t max_disk;   
        int64_t max_gpus;   
        int64_t min_cores;  
        int64_t min_memory; 
        int64_t min_disk;   
        int64_t min_gpus;   
        int64_t inuse_cache; 
};
 
 
struct vine_task *vine_task_create(const char *full_command);
 
void vine_task_delete(struct vine_task *t);
 
void vine_task_set_command(struct vine_task *t, const char *cmd);
 
void vine_task_set_library_required(struct vine_task *t, const char *name);
 
const char *vine_task_get_library_required(struct vine_task *t);
 
void vine_task_set_library_provided(struct vine_task *t, const char *name);
 
const char *vine_task_get_library_provided(struct vine_task *t);
 
void vine_task_set_function_slots(struct vine_task *t, int nslots);
 
void vine_task_set_function_exec_mode_from_string(struct vine_task *t, const char *exec_mode);
 
int vine_task_add_input(struct vine_task *t, struct vine_file *f, const char *remote_name, vine_mount_flags_t flags);
 
int vine_task_add_output(struct vine_task *t, struct vine_file *f, const char *remote_name, vine_mount_flags_t flags);
 
void vine_task_set_retries(struct vine_task *t, int64_t max_retries);
 
void vine_task_set_max_forsaken(struct vine_task *t, int64_t max_retries);
 
void vine_task_set_memory(struct vine_task *t, int64_t memory);
 
void vine_task_set_disk(struct vine_task *t, int64_t disk);
 
void vine_task_set_cores(struct vine_task *t, int cores);
 
void vine_task_set_gpus(struct vine_task *t, int gpus);
 
void vine_task_set_time_end(struct vine_task *t, int64_t useconds);
 
void vine_task_set_time_start(struct vine_task *t, int64_t useconds);
 
void vine_task_set_time_max(struct vine_task *t, int64_t seconds);
 
void vine_task_set_time_min(struct vine_task *t, int64_t seconds);
 
void vine_task_set_tag(struct vine_task *t, const char *tag);
 
void vine_task_set_category(struct vine_task *t, const char *category);
 
void vine_task_add_feature(struct vine_task *t, const char *name);
 
void vine_task_set_priority(struct vine_task *t, double priority);
 
void vine_task_set_env_var(struct vine_task *t, const char *name, const char *value);
 
void vine_task_set_scheduler(struct vine_task *t, vine_schedule_t algorithm);
 
int vine_task_set_monitor_output(struct vine_task *t, const char *monitor_output);
 
const char *vine_task_get_state(struct vine_task *t);
 
const char *vine_task_get_command(struct vine_task *t);
 
const char *vine_task_get_tag(struct vine_task *t);
 
const char *vine_task_get_category(struct vine_task *t);
 
int vine_task_get_id(struct vine_task *t);
 
vine_result_t vine_task_get_result(struct vine_task *t);
 
const char *vine_result_string(vine_result_t result);
 
int vine_task_get_exit_code(struct vine_task *t);
 
const char *vine_task_get_stdout(struct vine_task *t);
 
const char *vine_task_get_addrport(struct vine_task *t);
 
const char *vine_task_get_hostname(struct vine_task *t);
 
int64_t vine_task_get_metric(struct vine_task *t, const char *name);
 
void vine_task_set_resources(struct vine_task *t, const struct rmsummary *rm);
 
const struct rmsummary *vine_task_get_resources(struct vine_task *t, const char *name);
 
int vine_task_set_snapshot_file(struct vine_task *t, struct vine_file *monitor_snapshot_file);
 
int vine_task_add_starch_package(struct vine_task *t, struct vine_file *f);
 
int vine_task_add_poncho_package(struct vine_task *t, struct vine_file *f);
 
int vine_task_add_execution_context(struct vine_task *t, struct vine_file *f);
 
/* Deprecated alias for vine_task_add_execution_context. */
int vine_task_add_environment(struct vine_task *t, struct vine_file *f);
 
 
 
const char *vine_file_contents(struct vine_file *f);
 
size_t vine_file_size(struct vine_file *f);
 
const char *vine_file_source(struct vine_file *f);
 
vine_file_type_t vine_file_type(struct vine_file *f);
 
int vine_file_replica_count(struct vine_manager *m, struct vine_file *f);
 
struct vine_file *vine_declare_file(
                struct vine_manager *m, const char *source, vine_cache_level_t cache, vine_file_flags_t flags);
 
struct vine_file *vine_declare_url(
                struct vine_manager *m, const char *url, vine_cache_level_t cache, vine_file_flags_t flags);
 
struct vine_file *vine_declare_xrootd(struct vine_manager *m, const char *source, struct vine_file *proxy,
                struct vine_file *env, vine_cache_level_t cache, vine_file_flags_t flags);
 
struct vine_file *vine_declare_chirp(struct vine_manager *m, const char *server, const char *source,
                struct vine_file *ticket, struct vine_file *env, vine_cache_level_t cache, vine_file_flags_t flags);
 
struct vine_file *vine_declare_temp(struct vine_manager *m);
 
struct vine_file *vine_declare_buffer(struct vine_manager *m, const char *buffer, size_t size, vine_cache_level_t cache,
                vine_file_flags_t flags);
 
struct vine_file *vine_declare_mini_task(struct vine_manager *m, struct vine_task *mini_task, const char *name,
                vine_cache_level_t cache, vine_file_flags_t flags);
 
struct vine_file *vine_declare_untar(
                struct vine_manager *m, struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags);
 
struct vine_file *vine_declare_poncho(
                struct vine_manager *m, struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags);
 
struct vine_file *vine_declare_starch(
                struct vine_manager *m, struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags);
 
void vine_file_set_mode( struct vine_file *f, int mode );
 
const char *vine_fetch_file(struct vine_manager *m, struct vine_file *f);
 
void vine_undeclare_file(struct vine_manager *m, struct vine_file *f);
 
int vine_prune_file(struct vine_manager *m, struct vine_file *f);
 
 
 
struct vine_manager *vine_create(int port);
 
struct vine_manager *vine_ssl_create(int port, const char *key, const char *cert);
 
void vine_delete(struct vine_manager *m);
 
int vine_submit(struct vine_manager *m, struct vine_task *t);
 
void vine_manager_install_library(struct vine_manager *m, struct vine_task *t, const char *name);
 
void vine_manager_remove_library(struct vine_manager *m, const char *name);
 
struct vine_task *vine_manager_find_library_template(struct vine_manager *m, const char *library_name);
 
struct vine_task *vine_wait(struct vine_manager *m, int timeout);
 
struct vine_task *vine_wait_for_tag(struct vine_manager *m, const char *tag, int timeout);
 
struct vine_task *vine_wait_for_task_id(struct vine_manager *m, int task_id, int timeout);
 
int vine_hungry(struct vine_manager *m);
 
int vine_empty(struct vine_manager *m);
 
int vine_port(struct vine_manager *m);
 
void vine_set_name(struct vine_manager *m, const char *name);
 
const char *vine_get_name(struct vine_manager *m);
 
int vine_enable_monitoring(struct vine_manager *m, int watchdog, int time_series);
 
int vine_enable_peer_transfers(struct vine_manager *m);
 
int vine_disable_peer_transfers(struct vine_manager *m);
 
int vine_enable_proportional_resources(struct vine_manager *m);
 
int vine_disable_proportional_resources(struct vine_manager *m);
 
int vine_set_task_id_min(struct vine_manager *m, int minid);
 
void vine_block_host(struct vine_manager *m, const char *hostname);
 
void vine_block_host_with_timeout(struct vine_manager *m, const char *hostname, time_t seconds);
 
void vine_unblock_host(struct vine_manager *m, const char *hostname);
 
void vine_unblock_all(struct vine_manager *m);
 
void vine_get_stats(struct vine_manager *m, struct vine_stats *s);
 
void vine_get_stats_category(struct vine_manager *m, const char *c, struct vine_stats *s);
 
char *vine_get_status(struct vine_manager *m, const char *request);
 
struct rmsummary **vine_summarize_workers(struct vine_manager *m);
 
void vine_set_bandwidth_limit(struct vine_manager *m, const char *bandwidth);
 
double vine_get_effective_bandwidth(struct vine_manager *m);
 
int vine_enable_disconnect_slow_workers(struct vine_manager *m, double multiplier);
 
int vine_enable_disconnect_slow_workers_category(struct vine_manager *m, const char *category, double multiplier);
 
int vine_set_draining_by_hostname(struct vine_manager *m, const char *hostname, int drain_flag);
 
int vine_set_category_mode(struct vine_manager *m, const char *category, vine_category_mode_t mode);
 
void vine_set_category_max_concurrent(struct vine_manager *m, const char *category, int max_concurrent);
 
int vine_enable_category_resource(struct vine_manager *m, const char *category, const char *resource, int autolabel);
 
void vine_set_scheduler(struct vine_manager *m, vine_schedule_t algorithm);
 
void vine_set_priority(struct vine_manager *m, int priority);
 
void vine_set_tasks_left_count(struct vine_manager *m, int ntasks);
 
void vine_set_catalog_servers(struct vine_manager *m, const char *hosts);
 
/* Send updates to the catalog server.
@param m A manager object
*/
void vine_update_catalog(struct vine_manager *m);
 
void vine_set_property(struct vine_manager *m, const char *name, const char *value);
 
int vine_cancel_by_task_id(struct vine_manager *m, int id);
 
int vine_cancel_by_task_tag(struct vine_manager *m, const char *tag);
 
int vine_cancel_all(struct vine_manager *m);
 
int vine_enable_debug_log(const char *logfile);
 
int vine_enable_perf_log(struct vine_manager *m, const char *logfile);
 
int vine_enable_transactions_log(struct vine_manager *m, const char *logfile);
 
int vine_enable_taskgraph_log(struct vine_manager *m, const char *logfile);
 
int vine_workers_shutdown(struct vine_manager *m, int n);
 
void vine_set_password(struct vine_manager *m, const char *password);
 
int vine_set_password_file(struct vine_manager *m, const char *file);
 
void vine_set_keepalive_interval(struct vine_manager *m, int interval);
 
void vine_set_keepalive_timeout(struct vine_manager *m, int timeout);
 
void vine_set_manager_preferred_connection(struct vine_manager *m, const char *preferred_connection);
 
int vine_tune(struct vine_manager *m, const char *name, double value);
 
void vine_set_resources_max(struct vine_manager *m, const struct rmsummary *rm);
 
void vine_set_resources_min(struct vine_manager *m, const struct rmsummary *rm);
 
void vine_set_category_resources_max(struct vine_manager *m, const char *category, const struct rmsummary *rm);
 
void vine_set_category_resources_min(struct vine_manager *m, const char *category, const struct rmsummary *rm);
 
void vine_set_category_first_allocation_guess(struct vine_manager *m, const char *category, const struct rmsummary *rm);
 
void vine_initialize_categories(struct vine_manager *m, struct rmsummary *max, const char *summaries_file);
 
void vine_set_runtime_info_path(const char *path);
 
void vine_set_runtime_info_template(const char *dir);
 
void vine_log_debug_app(struct vine_manager *m, const char *entry);
 
void vine_log_txn_app(struct vine_manager *m, const char *entry);
 
void vine_counters_print();
 
char *vine_version_string();
 
char *vine_get_path_log(struct vine_manager *m, const char *path);
 
char *vine_get_path_staging(struct vine_manager *m, const char *path);
 
char *vine_get_path_library_log(struct vine_manager *m, const char *path);
 
char *vine_get_path_cache(struct vine_manager *m, const char *path);
 
 
#endif