* @version $Id$
* @access public
* @license http://opensource.org/licenses/gpl-3.0.html
*/
/**
* api_multivalue_javascript
* @return void
*/
if(!function_exists('api_multivalue_javascript')) {
function api_multivalue_javascript() {
$js = '$("#reload-discover-api").click(function() {
$("#api-url").change();
return false;
});;
$("#sdi_value_type").change(function() {
let sdiType = $("#sdi_value_type").val();
if(sdiType == "unique") {
$("#api-value-field-cont").css("display", "block");
$("#api-multi-mapping-cont").css("display", "none");
} else {
$("#api-value-field-cont").css("display", "none");
$("#api-multi-mapping-cont").css("display", "block");
}
});
$("#sdi_value_type").change();
$("#api-url").change(function() {
let sdiType = $("#sdi_value_type").val();
let api_url = $("#api-url").val();
console.log(api_url);
console.log(sdiType);
let content = CKEDITOR.instances["comment"].getData(); // we get content from comment textarea
var rval = /{(.*?)}/g.exec(content); // select what is between curly brackets : rval[0] returns content with brackets, rval[1] without
if(sdiType == "multiple" && !rval) { // no predefined values are set, we notify the user and exit
alertify.error("' . addslashes(sprintf(_t('sdi', 'api-no-predefined-values'), '#attached_to_dashboard')) . '");
return false;
}
if(api_url.length < 7) { // api_url is empty
// we do not do anything now
return false;
} else {
// we send the request to discover fields
$.ajax({
method: "GET",
url: "../dashboard/api-harvest.php?token='.SECRET_KEY.'",
data: { apiURL: $("#api-url").val() },
dataType: "json",
}).done(function(data) {
if(data.status == 0) { // failure : we display a message
alertify.error(data.msg);
return false;
}
if(data.status == 1) { // success : we prepare data
selectBoxOptions("api-date-field", data.keys, true); // we replace date field
if(sdiType == "unique") {
selectBoxOptions("api-value-field", data.keys, true); // we replace date field
}
if(sdiType == "multiple") {
if($(".api-mapping-field").length == 0) { // we prepare fields only if no existing data is there
entries = rval[1].split(";");
// @todo : idéalement récuperer les valeurs déjà peuplées
$("#api-multi-mapping-cont").empty();
for(var i = 0; i < entries.length; i++) {
if(entries[i] != "") $("#api-multi-mapping-cont").append("
");
}
}
// we apply selectBox anyway
$( ".api-mapping-field").each(function( index ) {
// console.log(this.id);
selectBoxOptions(this.id, data.keys, true);
});
}
}
});
}
// console.log(content);
// console.log(rval);
});';
footerAddInlineJS($js);
}
}
/**
* unit_javascript_validation
* @return void
*/
if(!function_exists('unit_javascript_validation')) {
function unit_javascript_validation() {
$js = '$("#min_value, #max_value, #threshold_value").change(function() {
var minVal = $("#min_value").val().replace(",", ".");
var maxVal = $("#max_value").val().replace(",", ".");
var thresholdVal = $("#threshold_value").val().replace(",", ".");
var dashboardViz = $("#sdi_dashboard_viz").find(":selected").val();
console.log("min : " + minVal + " - max : " + maxVal + " - threshold : " + thresholdVal);
if(isNumeric(minVal) && isNumeric(maxVal) && maxVal <= minVal ) alertify.error("'._t("sdi","object_minmax_error").'");
// if thresold and min are provided, be sure thresold is superior to min
if(isNumeric(minVal) && isNumeric(thresholdVal) && thresholdVal < minVal ) alertify.error("'._t("sdi","object_minthreshold_error").'");
// if thresold and max are provided, be sure thresold is inferior to max
if(isNumeric(maxVal) && isNumeric(thresholdVal) && thresholdVal > maxVal ) alertify.error("'._t("sdi","object_maxthreshold_error").'");
// we add check if gauge
if(dashboardViz == "gauge") {
if(!isNumeric(maxVal) || !isNumeric(minVal) || !isNumeric(thresholdVal)) alertify.error("'.sprintf(_t('sdi','object_gauge_error1'), $GLOBALS['lang']['sdi']['select_dashboard_viz']['gauge']).'");
if(isNumeric(maxVal) && isNumeric(thresholdVal) && thresholdVal >= maxVal ) alertify.error("'.sprintf(_t('sdi','object_gauge_error2'), $GLOBALS['lang']['sdi']['select_dashboard_viz']['gauge']).'");
if(isNumeric(minVal) && isNumeric(thresholdVal) && thresholdVal <= minVal ) alertify.error("'.sprintf(_t('sdi','object_gauge_error3'), $GLOBALS['lang']['sdi']['select_dashboard_viz']['gauge']).'");
}
});';
footerAddInlineJS($js);
}
}
/**
* get_unique_indicators
* @param array $sdi
* @param boolean $noorphan
* @return array
*/
if(!function_exists('get_unique_indicators')) {
function get_unique_indicators($sdi, $noorphan = false) {
$uniques = [];
$cnt = 0;
if($noorphan) {
$levelExclude = 0;
} else {
$levelExclude = -999; // wich has no effect
}
if(!is_array($sdi)) return array('indicators' => [], 'count' => 0);
foreach ($sdi as $s) {
if(!in_array($s['sdii_id'], $uniques) && $s['sdii_level'] != $levelExclude) {
array_push($uniques, $s['sdii_id']);
$cnt++;
}
}
return array('indicators' => $uniques, 'count' => $cnt);
}
}
/**
* _format_results
* Format results all the same way, whatever it comes from distinct API with different formats
* Resulting array is like so : array('results' => array(0 => array( 'value' => 100, 'date' => '2022', 'something' => 'xxx'), 1 => array( 'value' => 100, 'date' => '2022', 'something' => 'xxx'));
* @param array $json_arr
* @return array
*/
if(!function_exists('_format_results')) {
function _format_results(array $json_arr) {
$a = [];
// Linea21 formatting
if(isset($json_arr['values'])) {
$json_arr['results'] = $json_arr['values'];
}
// arcgis : 'features' + 'attributes' field in loop
// geoserver : 'features' + 'properties' field in loop
if(isset($json_arr['features'])) {
$tmp = [];
foreach ($json_arr['features'] as $rec) {
if(isset($rec['attributes'])) array_push($tmp, $rec['attributes']); // arcgis
if(isset($rec['properties'])) array_push($tmp, $rec['properties']); // geoserver
}
$json_arr['results'] = $tmp;
}
// opendatasoft and koumoul (default query type) : 'results'
if(isset($json_arr['results'])) $a = $json_arr;
// put returned values in results entry if 'results' key does not exists
// for example, no 'results' key when using https://api.atmosud.org/cigale/ges/territoire?variable=prg100.3ges&code_territoire=200030195&bool_sources_additionnelles=false&perimetre=pcaet&all_years=true&format=json
if(count($json_arr) > 0 && !isset($json_arr['results'])) $a = array('results' => $json_arr);
// @todo for koumoul aggregation see : https://data.ademe.fr/data-fair/api/v1/datasets/investissements-d'avenir-projets/values_agg?field=Code_EPCI&format=json&metric=sum&metric_field=Co%C3%BBt_total_du_projet&agg_size=99&qs=Code_EPCI%3A%22200018166%22&size=99&select=Total_autoris%C3%A9%2CCo%C3%BBt_total_du_projet%2CCode_EPCI
// print_r($a);
return $a;
}
}
/**
* api_discover
* @param array $sdi
* @param boolean $initialImport
* @return boolean
*/
if(!function_exists('api_discover')) {
function api_discover($url, $oneRow = true) {
# https://docs.guzzlephp.org/en/5.3/quickstart.html
// use GuzzleHttp\Client;
// $client = new Client();
// @see https://stackoverflow.com/questions/20847633/limit-connecting-time-with-guzzle-http-php-client
// we add a timeout to client not to block script
if(defined('PROXY_PARAMS')) {
$client = new GuzzleHttp\Client(['timeout' => 3, 'connect_timeout' => 6, 'proxy' => PROXY_PARAMS]);
} else {
$client = new GuzzleHttp\Client(['timeout' => 3, 'connect_timeout' => 3]);
}
// we check if url is responding, if not we log the error
try {
$response = $client->get($url);
$code = $response->getStatusCode();
if($code == '200') {
// we convert json response to array
$records = json_decode($response->getBody(), true);
// print_r($records);
// we format retrieved json file if needed
$records = _format_results($records);
// print_r($records);
if($oneRow) return $records['results'][0];
else return $records['results'];
}
// if 404 exception is caught - others errors as well
} catch (Exception $e) {
logfile(LOG_MAINFILE, array('[API discover]', 'URL not responding', $url, $e->getMessage()));
return _t('sdi', 'api-no-response');
}
}
}
/**
* api_update_indicators
* @param array $sdi
* @param boolean $initialImport
* @return boolean
*/
if(!function_exists('api_update_indicators')) {
function api_update_indicators($sdi, $initialImport = false) {
global $sql_object;
$debug = false;
$scale_id = 1;
$rflag = 1; // 1 means 'success', 2 : 'partial import due to strange values', 3 : 'action failed'
# https://docs.guzzlephp.org/en/5.3/quickstart.html
// use GuzzleHttp\Client;
// $client = new Client();
// @see https://stackoverflow.com/questions/20847633/limit-connecting-time-with-guzzle-http-php-client
// @see https://docs.guzzlephp.org/en/5.3/clients.html?highlight=proxi
// we add a timeout to client not to block script
if(defined('PROXY_PARAMS')) {
$client = new GuzzleHttp\Client(['timeout' => 3, 'connect_timeout' => 6, 'proxy' => PROXY_PARAMS]);
} else {
$client = new GuzzleHttp\Client(['timeout' => 3, 'connect_timeout' => 3]);
}
foreach ($sdi as $indicator) {
// if API is enabled only
if($indicator['sdii_api_enabled'] == 'Y') {
// $initialImport flag is not used anymore
// we set $since value anyway, not only if initialImport
if(!empty($indicator['sdii_api_getvalues-since'])) $since = $indicator['sdii_api_getvalues-since'];
else $since='1900'; // we fake the date to import everything
// $sql_object -> DBQuery("DELETE FROM l21_sdi_value WHERE sdiv_sdi_info ='" . $indicator['sdii_id'] . "' AND sdiv_scale ='" .$scale_id."'");
// we check if url is responding, if not we log the error
try {
$response = $client->get($indicator['sdii_api_url']);
$code = $response->getStatusCode();
// no test on $indicator['sdii_api_getvalues'] == 'Y', because it is supposed to be done before calling the function
if($code == '200') {
// we convert json response to array
$records = json_decode($response->getBody(), true);
// print_r($records);
// we format retrieved json file if needed
$records = _format_results($records);
foreach ($records['results'] as $rec) {
if($debug) echo '
';
if($debug) echo '
Value from API :
';
if($debug) print_r($rec);
if($debug) echo '';
// we select last entered values and compare with frequency to see if we have to execute update routine or not
// for all cases (even initial import) - if set, we get the latest indicator value and date for the given scale
$lastV = $sql_object -> DBSelect(SQL_getlastInsertByPublicationDate($indicator['sdii_id'], $scale_id));
// if exists, it can override $since variable to prevent multiple imports on initial import - security
if(isset($lastV[0]['sdiv_value']) ) {
$lastDate = $lastV[0]['sdiv_date_published']; // return 2020-01-01
$dateSuffix = substr($lastDate,4);
}
else {
$lastDate = $since;
$dateSuffix = '-12-31';
}
// we get threshold from old values or default value, if null, replacing by empty string
if(isset($lastV[0]['sdiv_value']) && !is_null($lastV[0]['sdiv_threshold'])) {
$threshold = $lastV[0]['sdiv_threshold'];
} else {
if(is_numeric($indicator['sdii_threshold_value'])) $threshold = $indicator['sdii_threshold_value'];
else $threshold = '';
}
$format = "Y-m-d"; // date object format
// we create date object for comparison
// if only year is passed, we add month and day and create the object
if(strlen($rec[$indicator['sdii_api_datefield']]) == 4) {
$rec[$indicator['sdii_api_datefield']] .= $dateSuffix;
$recordDateFormatted = $rec[$indicator['sdii_api_datefield']];
$recordDate = DateTime::createFromFormat($format, $rec[$indicator['sdii_api_datefield']]);
// else default format is similar to : "2012-01-01T00:00:00+00:00"
} else {
$recordDateFormatted = $rec[$indicator['sdii_api_datefield']];
$recordDate = new DateTime($rec[$indicator['sdii_api_datefield']]);
}
// we create a formatted date for display
$formattedDate = $recordDate->format("Y-m-d"); // Output format : 2023-06-04
// only year is passed to since, we add month and day
if(strlen($since) == 4) $since .= $dateSuffix;
if(strlen($lastDate) == 4) $lastDate .= $dateSuffix;
$SinceDate = DateTime::createFromFormat($format, $since);
// we get last date
$lastvalue = DateTime::createFromFormat($format, $lastDate);
$formattedLastvalue = $lastvalue->format("Y-m-d");
// we add interval to latest retrieved date
$triggerDate = $lastvalue;
$triggerDate->modify('+' . $indicator['sdii_frequency'] . 'days');
// we finally compare $triggerDate with $recordDate to check if we have to add records in database
if($debug) echo "
Indicator id : ".$indicator['sdii_id'] ." - Since date : " . $SinceDate->format("Y-m-d") . " / [comparison] current record \$recordate (from API) : " . $formattedDate . ' / last value\'s date (from DB) : ' . $formattedLastvalue . " / trigger date \$triggerdate (from DB) - pour ajout seulement : " . $triggerDate->format("Y-m-d") . ' / frequency : '.$indicator['sdii_frequency'] . " / overwrite values : " . $indicator['sdii_api_overwritevalues'] . '
';
// if($indicator['sdii_api_overwritevalues'] == 'N' && $recordDate >= $triggerDate) {
// be sure we import values for desired years
if($recordDate >= $SinceDate) {
if(isset($rec[$indicator['sdii_api_datefield']])) {
$a = [];
$a['indicator_type'] = $indicator['sdii_value_type'];
// if is sdii_value_type == 'unique'
// !!!! WARNING : if( isset($rec[$indicator['sdii_api_valuefield']])) does not pass test
// when $rec[$indicator['sdii_api_valuefield']] == null
if ($indicator['sdii_value_type'] == 'unique') {
if (is_numeric($rec[$indicator['sdii_api_valuefield']])) {
$a[0] = sys_number_format(round($rec[$indicator['sdii_api_valuefield']], $indicator['sdii_api_decimal']));
$a['nodata'] = 0;
} else {
$a[0] = 0;
$a['nodata'] = 1;
}
}
// if is sdii_value_type == 'multiple'
if ($indicator['sdii_value_type'] == 'multiple') {
$mvalues = [];
$mapping = unserialize($indicator['sdii_api_multimapping']);
// first iteration to be sure data exists and is numeric
$flags = [];
foreach ($mapping as $k => $v) {
if (isset($rec[$v]) && is_numeric($rec[$v])) array_push($flags, true);
else array_push($flags, false); // we handle
}
// Multiple values format - no errors, we populate records
// Array ( [0] => Array ( [value] => 45 [label] => chaleur ) [1] => Array ( [value] => 89 [label] => électricité ) [2] => Array ( [value] => 52 [label] => carburant ) )
if (!in_array(false, $flags)) {
foreach ($mapping as $k => $v) {
array_push($mvalues, array('value' => sys_number_format(round($rec[$v], $indicator['sdii_api_decimal'])), 'label' => $k));
}
$a['nodata'] = 0;
} else {
foreach ($mapping as $k => $v) {
array_push($mvalues, array('value' => 0, 'label' => $k));
}
$a['nodata'] = 1;
}
$a[0] = $mvalues;
}
$a[1] = $indicator['sdii_id']; // indicator id
$a[2] = $threshold;
$a[3] = $scale_id; // scale id
$a[4] = $formattedDate; // date (format yyyy-mm-dd)
$a[5] = 'p'; // status
$a[6] = _t('sdi', 'api-retrieved'); // comment
$a[7] = 'N'; // comment status
$a['user_id'] = 0; // 0 stands for API
$sdio = new sdi;
$integrity = $sdio->CheckDataIntegrity_value($a, $sql_object);
// Indicator is not set to update values, we just add new values if exists
if ($indicator['sdii_api_overwritevalues'] == 'N' && $recordDate >= $triggerDate) {
if (!is_string($integrity)) {
if ($debug) echo '
ADD VALUE / integrity : ' . $integrity. '
';
$r = $sdio->AddValue($a, $sql_object);
if (is_numeric($r)) logfile(LOG_MAINFILE, array('[API call]', 'SUCCESS - adding values to db', 'indicator : ' . $indicator['sdii_id'], 'added value ID : ' . $r));
} else {
// CheckDataIntegrity_value failed
logfile(LOG_MAINFILE, array('[API call]', 'ERROR - adding values / CheckDataIntegrity_value() failed', 'indicator : ' . $indicator['sdii_id'], $indicator['sdii_api_url'], 'error message : ' . strip_tags((string)$integrity)));
// return false; // we do no return false to still execute the routine on dataset
$rflag = 2; // partial import due to strange values
}
}
// Indicator is set to update values, we do it only if value has changed for a given date
if ($indicator['sdii_api_overwritevalues'] == 'Y') {
$res = $sql_object->DBSelect(SQL_getIndicValuesBydate($indicator['sdii_id'], $scale_id, $recordDateFormatted));
// check if value has changed for a given date and modify value if needed
if(value_has_changed($a, $res, $indicator)) {
if (!is_string($integrity)) {
// value already exists we modify it if needed !
if(isset($res[0]['sdiv_id'])) {
if ($debug) echo '
UPDATE VALUE / integrity : ' . $integrity. '
';
$r = $sdio->ModifyValue($res[0]['sdiv_id'], $a, $sql_object);
if (is_numeric($r)) logfile(LOG_MAINFILE, array('[API call]', 'SUCCESS - modifying values to db', 'indicator : ' . $indicator['sdii_id'], 'added value ID : ' . $r));
} elseif(!isset($res[0]['sdiv_id']) && $recordDate >= $triggerDate) {
if ($debug) echo '
ADD VALUE / integrity : ' . $integrity. '
';
$r = $sdio->AddValue($a, $sql_object);
if (is_numeric($r)) logfile(LOG_MAINFILE, array('[API call]', 'SUCCESS - adding values to db', 'indicator : ' . $indicator['sdii_id'], 'added value ID : ' . $r));
}
} else {
if(isset($res[0]['sdiv_id'])) {
// CheckDataIntegrity_value failed
logfile(LOG_MAINFILE, array('[API call]', 'ERROR - modifying values / CheckDataIntegrity_value() failed', 'indicator : ' . $indicator['sdii_id'], $res[0]['sdiv_id'], $indicator['sdii_api_url'], 'error message : ' . strip_tags((string)$integrity)));
} else {
logfile(LOG_MAINFILE, array('[API call]', 'ERROR - adding values / CheckDataIntegrity_value() failed', 'indicator : ' . $indicator['sdii_id'], $indicator['sdii_api_url'], 'error message : ' . strip_tags((string)$integrity)));
}
// return false; // we do no return false to still execute the routine on dataset
$rflag = 2; // partial import due to strange values
}
}
}
} else {
// given keys are inconsistent
logfile(LOG_MAINFILE, array('[API call]', 'ERROR - given array keys are inconsistent : ' . $indicator['sdii_api_valuefield'] . ' / "'. $indicator['sdii_api_datefield']. ' - check url result and change fields values', 'indicator : ' . $indicator['sdii_id'], $indicator['sdii_api_url']));
$rflag = 2; // partial import due to strange values
// return false; // we do no return false to still execute the routine on dataset
}
}
if($debug) echo '
'; // closing debug container
}
}
// if 404 exception is caught - others errors as well
} catch (Exception $e) {
logfile(LOG_MAINFILE, array('[API call]', 'URL not responding', $indicator['sdii_api_url'], 'indicator : ' . $indicator['sdii_id'], $e->getMessage()));
$rflag = 3;
}
} // closing $indicator['sdii_api_enabled'] == 'Y' test
}
return $rflag; // 1, 2 or 3 here
}
}
if(!function_exists('value_has_changed')) {
function value_has_changed($a, $res, $indicator) {
$debug = false;
// no record found for the given date, so no modification
if(!is_array($res)) {
if ($debug) echo '
No values is stored for the current year. So, no changes !
';
return true;
}
// for unique type indicator
if($indicator['sdii_value_type'] == 'unique') {
if($debug) {
echo '
'.$res[0]['sdiv_date_published'].' - Value in db : '.sys_number_format(round($res[0]['sdiv_value'], $indicator['sdii_api_decimal'])).' / Value from API '.sys_number_format(round($a[0], $indicator['sdii_api_decimal'])).'
';
if (sys_number_format(round($res[0]['sdiv_value'], $indicator['sdii_api_decimal'])) == sys_number_format(round($a[0], $indicator['sdii_api_decimal']))) echo '
No changes
';
else echo '
Changes ! Update is needed !
';
}
if($res[0]['sdiv_value'] != $a[0]) return true;
}
// for multiple type indicator
if($indicator['sdii_value_type'] == 'multiple') {
$mapping = unserialize($indicator['sdii_api_multimapping']);
// we format stored values to be able to compared retrieved values with existing ones
$tmp = [];
$currentval = unserialize ($res[0]['sdiv_multivalue']);
foreach ($currentval as $k => $v) {
array_push($tmp, ['value' => $v['value'], 'label' => $v['label']]);
}
if($debug) {
echo '
Formatted values from database : ';
print_r($tmp);
echo '
';
echo '
Formatted values from API : ';
print_r($a[0]);
echo '
';
// there is no value
if ($indicatorValues==false) {
$str .= '-';
// some values are there
} else {
$data = array();
$class = '';
// we reverse the array to display eldest first
$values = array_reverse($indicatorValues);
if(isset($indicatorValues[0]['sdiv_cumulative_value'])) $cumulative_value = $indicatorValues[0]['sdiv_cumulative_value'];
foreach($values as $value) {
// we handle multivalues
if(!empty($value['sdiv_multivalue'])) {
$dataindic = unserialize($value['sdiv_multivalue']);
array_push($data, $dataindic[0][$cfield]);
// we handle simple values
} else {
array_push($data, $value['sdiv_value']);
}
}
$str .= '