diff --git a/_freeze/site_libs/crosstalk-1.2.1/css/crosstalk.min.css b/_freeze/site_libs/crosstalk-1.2.1/css/crosstalk.min.css new file mode 100644 index 00000000..6b453828 --- /dev/null +++ b/_freeze/site_libs/crosstalk-1.2.1/css/crosstalk.min.css @@ -0,0 +1 @@ +.container-fluid.crosstalk-bscols{margin-left:-30px;margin-right:-30px;white-space:normal}body>.container-fluid.crosstalk-bscols{margin-left:auto;margin-right:auto}.crosstalk-input-checkboxgroup .crosstalk-options-group .crosstalk-options-column{display:inline-block;padding-right:12px;vertical-align:top}@media only screen and (max-width: 480px){.crosstalk-input-checkboxgroup .crosstalk-options-group .crosstalk-options-column{display:block;padding-right:inherit}}.crosstalk-input{margin-bottom:15px}.crosstalk-input .control-label{margin-bottom:0;vertical-align:middle}.crosstalk-input input[type="checkbox"]{margin:4px 0 0;margin-top:1px;line-height:normal}.crosstalk-input .checkbox{position:relative;display:block;margin-top:10px;margin-bottom:10px}.crosstalk-input .checkbox>label{padding-left:20px;margin-bottom:0;font-weight:400;cursor:pointer}.crosstalk-input .checkbox input[type="checkbox"],.crosstalk-input .checkbox-inline input[type="checkbox"]{position:absolute;margin-top:2px;margin-left:-20px}.crosstalk-input .checkbox+.checkbox{margin-top:-5px}.crosstalk-input .checkbox-inline{position:relative;display:inline-block;padding-left:20px;margin-bottom:0;font-weight:400;vertical-align:middle;cursor:pointer}.crosstalk-input .checkbox-inline+.checkbox-inline{margin-top:0;margin-left:10px} diff --git a/_freeze/site_libs/crosstalk-1.2.1/js/crosstalk.js b/_freeze/site_libs/crosstalk-1.2.1/js/crosstalk.js new file mode 100644 index 00000000..fd9eb53d --- /dev/null +++ b/_freeze/site_libs/crosstalk-1.2.1/js/crosstalk.js @@ -0,0 +1,1474 @@ +(function(){function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o b) { + return 1; + } +} + +/** + * @private + */ + +var FilterSet = function () { + function FilterSet() { + _classCallCheck(this, FilterSet); + + this.reset(); + } + + _createClass(FilterSet, [{ + key: "reset", + value: function reset() { + // Key: handle ID, Value: array of selected keys, or null + this._handles = {}; + // Key: key string, Value: count of handles that include it + this._keys = {}; + this._value = null; + this._activeHandles = 0; + } + }, { + key: "update", + value: function update(handleId, keys) { + if (keys !== null) { + keys = keys.slice(0); // clone before sorting + keys.sort(naturalComparator); + } + + var _diffSortedLists = (0, _util.diffSortedLists)(this._handles[handleId], keys), + added = _diffSortedLists.added, + removed = _diffSortedLists.removed; + + this._handles[handleId] = keys; + + for (var i = 0; i < added.length; i++) { + this._keys[added[i]] = (this._keys[added[i]] || 0) + 1; + } + for (var _i = 0; _i < removed.length; _i++) { + this._keys[removed[_i]]--; + } + + this._updateValue(keys); + } + + /** + * @param {string[]} keys Sorted array of strings that indicate + * a superset of possible keys. + * @private + */ + + }, { + key: "_updateValue", + value: function _updateValue() { + var keys = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : this._allKeys; + + var handleCount = Object.keys(this._handles).length; + if (handleCount === 0) { + this._value = null; + } else { + this._value = []; + for (var i = 0; i < keys.length; i++) { + var count = this._keys[keys[i]]; + if (count === handleCount) { + this._value.push(keys[i]); + } + } + } + } + }, { + key: "clear", + value: function clear(handleId) { + if (typeof this._handles[handleId] === "undefined") { + return; + } + + var keys = this._handles[handleId]; + if (!keys) { + keys = []; + } + + for (var i = 0; i < keys.length; i++) { + this._keys[keys[i]]--; + } + delete this._handles[handleId]; + + this._updateValue(); + } + }, { + key: "value", + get: function get() { + return this._value; + } + }, { + key: "_allKeys", + get: function get() { + var allKeys = Object.keys(this._keys); + allKeys.sort(naturalComparator); + return allKeys; + } + }]); + + return FilterSet; +}(); + +exports.default = FilterSet; + +},{"./util":11}],4:[function(require,module,exports){ +(function (global){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; + +exports.default = group; + +var _var2 = require("./var"); + +var _var3 = _interopRequireDefault(_var2); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +// Use a global so that multiple copies of crosstalk.js can be loaded and still +// have groups behave as singletons across all copies. +global.__crosstalk_groups = global.__crosstalk_groups || {}; +var groups = global.__crosstalk_groups; + +function group(groupName) { + if (groupName && typeof groupName === "string") { + if (!groups.hasOwnProperty(groupName)) { + groups[groupName] = new Group(groupName); + } + return groups[groupName]; + } else if ((typeof groupName === "undefined" ? "undefined" : _typeof(groupName)) === "object" && groupName._vars && groupName.var) { + // Appears to already be a group object + return groupName; + } else if (Array.isArray(groupName) && groupName.length == 1 && typeof groupName[0] === "string") { + return group(groupName[0]); + } else { + throw new Error("Invalid groupName argument"); + } +} + +var Group = function () { + function Group(name) { + _classCallCheck(this, Group); + + this.name = name; + this._vars = {}; + } + + _createClass(Group, [{ + key: "var", + value: function _var(name) { + if (!name || typeof name !== "string") { + throw new Error("Invalid var name"); + } + + if (!this._vars.hasOwnProperty(name)) this._vars[name] = new _var3.default(this, name); + return this._vars[name]; + } + }, { + key: "has", + value: function has(name) { + if (!name || typeof name !== "string") { + throw new Error("Invalid var name"); + } + + return this._vars.hasOwnProperty(name); + } + }]); + + return Group; +}(); + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./var":12}],5:[function(require,module,exports){ +(function (global){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _group = require("./group"); + +var _group2 = _interopRequireDefault(_group); + +var _selection = require("./selection"); + +var _filter = require("./filter"); + +var _input = require("./input"); + +require("./input_selectize"); + +require("./input_checkboxgroup"); + +require("./input_slider"); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +var defaultGroup = (0, _group2.default)("default"); + +function var_(name) { + return defaultGroup.var(name); +} + +function has(name) { + return defaultGroup.has(name); +} + +if (global.Shiny) { + global.Shiny.addCustomMessageHandler("update-client-value", function (message) { + if (typeof message.group === "string") { + (0, _group2.default)(message.group).var(message.name).set(message.value); + } else { + var_(message.name).set(message.value); + } + }); +} + +var crosstalk = { + group: _group2.default, + var: var_, + has: has, + SelectionHandle: _selection.SelectionHandle, + FilterHandle: _filter.FilterHandle, + bind: _input.bind +}; + +/** + * @namespace crosstalk + */ +exports.default = crosstalk; + +global.crosstalk = crosstalk; + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./filter":2,"./group":4,"./input":6,"./input_checkboxgroup":7,"./input_selectize":8,"./input_slider":9,"./selection":10}],6:[function(require,module,exports){ +(function (global){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.register = register; +exports.bind = bind; +var $ = global.jQuery; + +var bindings = {}; + +function register(reg) { + bindings[reg.className] = reg; + if (global.document && global.document.readyState !== "complete") { + $(function () { + bind(); + }); + } else if (global.document) { + setTimeout(bind, 100); + } +} + +function bind() { + Object.keys(bindings).forEach(function (className) { + var binding = bindings[className]; + $("." + binding.className).not(".crosstalk-input-bound").each(function (i, el) { + bindInstance(binding, el); + }); + }); +} + +// Escape jQuery identifier +function $escape(val) { + return val.replace(/([!"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, "\\$1"); +} + +function bindEl(el) { + var $el = $(el); + Object.keys(bindings).forEach(function (className) { + if ($el.hasClass(className) && !$el.hasClass("crosstalk-input-bound")) { + var binding = bindings[className]; + bindInstance(binding, el); + } + }); +} + +function bindInstance(binding, el) { + var jsonEl = $(el).find("script[type='application/json'][data-for='" + $escape(el.id) + "']"); + var data = JSON.parse(jsonEl[0].innerText); + + var instance = binding.factory(el, data); + $(el).data("crosstalk-instance", instance); + $(el).addClass("crosstalk-input-bound"); +} + +if (global.Shiny) { + var inputBinding = new global.Shiny.InputBinding(); + var _$ = global.jQuery; + _$.extend(inputBinding, { + find: function find(scope) { + return _$(scope).find(".crosstalk-input"); + }, + initialize: function initialize(el) { + if (!_$(el).hasClass("crosstalk-input-bound")) { + bindEl(el); + } + }, + getId: function getId(el) { + return el.id; + }, + getValue: function getValue(el) {}, + setValue: function setValue(el, value) {}, + receiveMessage: function receiveMessage(el, data) {}, + subscribe: function subscribe(el, callback) { + _$(el).data("crosstalk-instance").resume(); + }, + unsubscribe: function unsubscribe(el) { + _$(el).data("crosstalk-instance").suspend(); + } + }); + global.Shiny.inputBindings.register(inputBinding, "crosstalk.inputBinding"); +} + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{}],7:[function(require,module,exports){ +(function (global){ +"use strict"; + +var _input = require("./input"); + +var input = _interopRequireWildcard(_input); + +var _filter = require("./filter"); + +function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } } + +var $ = global.jQuery; + +input.register({ + className: "crosstalk-input-checkboxgroup", + + factory: function factory(el, data) { + /* + * map: {"groupA": ["keyA", "keyB", ...], ...} + * group: "ct-groupname" + */ + var ctHandle = new _filter.FilterHandle(data.group); + + var lastKnownKeys = void 0; + var $el = $(el); + $el.on("change", "input[type='checkbox']", function () { + var checked = $el.find("input[type='checkbox']:checked"); + if (checked.length === 0) { + lastKnownKeys = null; + ctHandle.clear(); + } else { + var keys = {}; + checked.each(function () { + data.map[this.value].forEach(function (key) { + keys[key] = true; + }); + }); + var keyArray = Object.keys(keys); + keyArray.sort(); + lastKnownKeys = keyArray; + ctHandle.set(keyArray); + } + }); + + return { + suspend: function suspend() { + ctHandle.clear(); + }, + resume: function resume() { + if (lastKnownKeys) ctHandle.set(lastKnownKeys); + } + }; + } +}); + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./filter":2,"./input":6}],8:[function(require,module,exports){ +(function (global){ +"use strict"; + +var _input = require("./input"); + +var input = _interopRequireWildcard(_input); + +var _util = require("./util"); + +var util = _interopRequireWildcard(_util); + +var _filter = require("./filter"); + +function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } } + +var $ = global.jQuery; + +input.register({ + className: "crosstalk-input-select", + + factory: function factory(el, data) { + /* + * items: {value: [...], label: [...]} + * map: {"groupA": ["keyA", "keyB", ...], ...} + * group: "ct-groupname" + */ + + var first = [{ value: "", label: "(All)" }]; + var items = util.dataframeToD3(data.items); + var opts = { + options: first.concat(items), + valueField: "value", + labelField: "label", + searchField: "label" + }; + + var select = $(el).find("select")[0]; + + var selectize = $(select).selectize(opts)[0].selectize; + + var ctHandle = new _filter.FilterHandle(data.group); + + var lastKnownKeys = void 0; + selectize.on("change", function () { + if (selectize.items.length === 0) { + lastKnownKeys = null; + ctHandle.clear(); + } else { + var keys = {}; + selectize.items.forEach(function (group) { + data.map[group].forEach(function (key) { + keys[key] = true; + }); + }); + var keyArray = Object.keys(keys); + keyArray.sort(); + lastKnownKeys = keyArray; + ctHandle.set(keyArray); + } + }); + + return { + suspend: function suspend() { + ctHandle.clear(); + }, + resume: function resume() { + if (lastKnownKeys) ctHandle.set(lastKnownKeys); + } + }; + } +}); + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./filter":2,"./input":6,"./util":11}],9:[function(require,module,exports){ +(function (global){ +"use strict"; + +var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }(); + +var _input = require("./input"); + +var input = _interopRequireWildcard(_input); + +var _filter = require("./filter"); + +function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } } + +var $ = global.jQuery; +var strftime = global.strftime; + +input.register({ + className: "crosstalk-input-slider", + + factory: function factory(el, data) { + /* + * map: {"groupA": ["keyA", "keyB", ...], ...} + * group: "ct-groupname" + */ + var ctHandle = new _filter.FilterHandle(data.group); + + var opts = {}; + var $el = $(el).find("input"); + var dataType = $el.data("data-type"); + var timeFormat = $el.data("time-format"); + var round = $el.data("round"); + var timeFormatter = void 0; + + // Set up formatting functions + if (dataType === "date") { + timeFormatter = strftime.utc(); + opts.prettify = function (num) { + return timeFormatter(timeFormat, new Date(num)); + }; + } else if (dataType === "datetime") { + var timezone = $el.data("timezone"); + if (timezone) timeFormatter = strftime.timezone(timezone);else timeFormatter = strftime; + + opts.prettify = function (num) { + return timeFormatter(timeFormat, new Date(num)); + }; + } else if (dataType === "number") { + if (typeof round !== "undefined") opts.prettify = function (num) { + var factor = Math.pow(10, round); + return Math.round(num * factor) / factor; + }; + } + + $el.ionRangeSlider(opts); + + function getValue() { + var result = $el.data("ionRangeSlider").result; + + // Function for converting numeric value from slider to appropriate type. + var convert = void 0; + var dataType = $el.data("data-type"); + if (dataType === "date") { + convert = function convert(val) { + return formatDateUTC(new Date(+val)); + }; + } else if (dataType === "datetime") { + convert = function convert(val) { + // Convert ms to s + return +val / 1000; + }; + } else { + convert = function convert(val) { + return +val; + }; + } + + if ($el.data("ionRangeSlider").options.type === "double") { + return [convert(result.from), convert(result.to)]; + } else { + return convert(result.from); + } + } + + var lastKnownKeys = null; + + $el.on("change.crosstalkSliderInput", function (event) { + if (!$el.data("updating") && !$el.data("animating")) { + var _getValue = getValue(), + _getValue2 = _slicedToArray(_getValue, 2), + from = _getValue2[0], + to = _getValue2[1]; + + var keys = []; + for (var i = 0; i < data.values.length; i++) { + var val = data.values[i]; + if (val >= from && val <= to) { + keys.push(data.keys[i]); + } + } + keys.sort(); + ctHandle.set(keys); + lastKnownKeys = keys; + } + }); + + // let $el = $(el); + // $el.on("change", "input[type="checkbox"]", function() { + // let checked = $el.find("input[type="checkbox"]:checked"); + // if (checked.length === 0) { + // ctHandle.clear(); + // } else { + // let keys = {}; + // checked.each(function() { + // data.map[this.value].forEach(function(key) { + // keys[key] = true; + // }); + // }); + // let keyArray = Object.keys(keys); + // keyArray.sort(); + // ctHandle.set(keyArray); + // } + // }); + + return { + suspend: function suspend() { + ctHandle.clear(); + }, + resume: function resume() { + if (lastKnownKeys) ctHandle.set(lastKnownKeys); + } + }; + } +}); + +// Convert a number to a string with leading zeros +function padZeros(n, digits) { + var str = n.toString(); + while (str.length < digits) { + str = "0" + str; + }return str; +} + +// Given a Date object, return a string in yyyy-mm-dd format, using the +// UTC date. This may be a day off from the date in the local time zone. +function formatDateUTC(date) { + if (date instanceof Date) { + return date.getUTCFullYear() + "-" + padZeros(date.getUTCMonth() + 1, 2) + "-" + padZeros(date.getUTCDate(), 2); + } else { + return null; + } +} + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./filter":2,"./input":6}],10:[function(require,module,exports){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.SelectionHandle = undefined; + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _events = require("./events"); + +var _events2 = _interopRequireDefault(_events); + +var _group = require("./group"); + +var _group2 = _interopRequireDefault(_group); + +var _util = require("./util"); + +var util = _interopRequireWildcard(_util); + +function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } } + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +/** + * Use this class to read and write (and listen for changes to) the selection + * for a Crosstalk group. This is intended to be used for linked brushing. + * + * If two (or more) `SelectionHandle` instances in the same webpage share the + * same group name, they will share the same state. Setting the selection using + * one `SelectionHandle` instance will result in the `value` property instantly + * changing across the others, and `"change"` event listeners on all instances + * (including the one that initiated the sending) will fire. + * + * @param {string} [group] - The name of the Crosstalk group, or if none, + * null or undefined (or any other falsy value). This can be changed later + * via the [SelectionHandle#setGroup](#setGroup) method. + * @param {Object} [extraInfo] - An object whose properties will be copied to + * the event object whenever an event is emitted. + */ +var SelectionHandle = exports.SelectionHandle = function () { + function SelectionHandle() { + var group = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : null; + var extraInfo = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : null; + + _classCallCheck(this, SelectionHandle); + + this._eventRelay = new _events2.default(); + this._emitter = new util.SubscriptionTracker(this._eventRelay); + + // Name of the group we're currently tracking, if any. Can change over time. + this._group = null; + // The Var we're currently tracking, if any. Can change over time. + this._var = null; + // The event handler subscription we currently have on var.on("change"). + this._varOnChangeSub = null; + + this._extraInfo = util.extend({ sender: this }, extraInfo); + + this.setGroup(group); + } + + /** + * Changes the Crosstalk group membership of this SelectionHandle. The group + * being switched away from (if any) will not have its selection value + * modified as a result of calling `setGroup`, even if this handle was the + * most recent handle to set the selection of the group. + * + * The group being switched to (if any) will also not have its selection value + * modified as a result of calling `setGroup`. If you want to set the + * selection value of the new group, call `set` explicitly. + * + * @param {string} group - The name of the Crosstalk group, or null (or + * undefined) to clear the group. + */ + + + _createClass(SelectionHandle, [{ + key: "setGroup", + value: function setGroup(group) { + var _this = this; + + // If group is unchanged, do nothing + if (this._group === group) return; + // Treat null, undefined, and other falsy values the same + if (!this._group && !group) return; + + if (this._var) { + this._var.off("change", this._varOnChangeSub); + this._var = null; + this._varOnChangeSub = null; + } + + this._group = group; + + if (group) { + this._var = (0, _group2.default)(group).var("selection"); + var sub = this._var.on("change", function (e) { + _this._eventRelay.trigger("change", e, _this); + }); + this._varOnChangeSub = sub; + } + } + + /** + * Retrieves the current selection for the group represented by this + * `SelectionHandle`. + * + * - If no selection is active, then this value will be falsy. + * - If a selection is active, but no data points are selected, then this + * value will be an empty array. + * - If a selection is active, and data points are selected, then the keys + * of the selected data points will be present in the array. + */ + + }, { + key: "_mergeExtraInfo", + + + /** + * Combines the given `extraInfo` (if any) with the handle's default + * `_extraInfo` (if any). + * @private + */ + value: function _mergeExtraInfo(extraInfo) { + // Important incidental effect: shallow clone is returned + return util.extend({}, this._extraInfo ? this._extraInfo : null, extraInfo ? extraInfo : null); + } + + /** + * Overwrites the current selection for the group, and raises the `"change"` + * event among all of the group's '`SelectionHandle` instances (including + * this one). + * + * @fires SelectionHandle#change + * @param {string[]} selectedKeys - Falsy, empty array, or array of keys (see + * {@link SelectionHandle#value}). + * @param {Object} [extraInfo] - Extra properties to be included on the event + * object that's passed to listeners (in addition to any options that were + * passed into the `SelectionHandle` constructor). + */ + + }, { + key: "set", + value: function set(selectedKeys, extraInfo) { + if (this._var) this._var.set(selectedKeys, this._mergeExtraInfo(extraInfo)); + } + + /** + * Overwrites the current selection for the group, and raises the `"change"` + * event among all of the group's '`SelectionHandle` instances (including + * this one). + * + * @fires SelectionHandle#change + * @param {Object} [extraInfo] - Extra properties to be included on the event + * object that's passed to listeners (in addition to any that were passed + * into the `SelectionHandle` constructor). + */ + + }, { + key: "clear", + value: function clear(extraInfo) { + if (this._var) this.set(void 0, this._mergeExtraInfo(extraInfo)); + } + + /** + * Subscribes to events on this `SelectionHandle`. + * + * @param {string} eventType - Indicates the type of events to listen to. + * Currently, only `"change"` is supported. + * @param {SelectionHandle~listener} listener - The callback function that + * will be invoked when the event occurs. + * @return {string} - A token to pass to {@link SelectionHandle#off} to cancel + * this subscription. + */ + + }, { + key: "on", + value: function on(eventType, listener) { + return this._emitter.on(eventType, listener); + } + + /** + * Cancels event subscriptions created by {@link SelectionHandle#on}. + * + * @param {string} eventType - The type of event to unsubscribe. + * @param {string|SelectionHandle~listener} listener - Either the callback + * function previously passed into {@link SelectionHandle#on}, or the + * string that was returned from {@link SelectionHandle#on}. + */ + + }, { + key: "off", + value: function off(eventType, listener) { + return this._emitter.off(eventType, listener); + } + + /** + * Shuts down the `SelectionHandle` object. + * + * Removes all event listeners that were added through this handle. + */ + + }, { + key: "close", + value: function close() { + this._emitter.removeAllListeners(); + this.setGroup(null); + } + }, { + key: "value", + get: function get() { + return this._var ? this._var.get() : null; + } + }]); + + return SelectionHandle; +}(); + +/** + * @callback SelectionHandle~listener + * @param {Object} event - An object containing details of the event. For + * `"change"` events, this includes the properties `value` (the new + * value of the selection, or `undefined` if no selection is active), + * `oldValue` (the previous value of the selection), and `sender` (the + * `SelectionHandle` instance that made the change). + */ + +/** + * @event SelectionHandle#change + * @type {object} + * @property {object} value - The new value of the selection, or `undefined` + * if no selection is active. + * @property {object} oldValue - The previous value of the selection. + * @property {SelectionHandle} sender - The `SelectionHandle` instance that + * changed the value. + */ + +},{"./events":1,"./group":4,"./util":11}],11:[function(require,module,exports){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; + +exports.extend = extend; +exports.checkSorted = checkSorted; +exports.diffSortedLists = diffSortedLists; +exports.dataframeToD3 = dataframeToD3; + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +function extend(target) { + for (var _len = arguments.length, sources = Array(_len > 1 ? _len - 1 : 0), _key = 1; _key < _len; _key++) { + sources[_key - 1] = arguments[_key]; + } + + for (var i = 0; i < sources.length; i++) { + var src = sources[i]; + if (typeof src === "undefined" || src === null) continue; + + for (var key in src) { + if (src.hasOwnProperty(key)) { + target[key] = src[key]; + } + } + } + return target; +} + +function checkSorted(list) { + for (var i = 1; i < list.length; i++) { + if (list[i] <= list[i - 1]) { + throw new Error("List is not sorted or contains duplicate"); + } + } +} + +function diffSortedLists(a, b) { + var i_a = 0; + var i_b = 0; + + if (!a) a = []; + if (!b) b = []; + + var a_only = []; + var b_only = []; + + checkSorted(a); + checkSorted(b); + + while (i_a < a.length && i_b < b.length) { + if (a[i_a] === b[i_b]) { + i_a++; + i_b++; + } else if (a[i_a] < b[i_b]) { + a_only.push(a[i_a++]); + } else { + b_only.push(b[i_b++]); + } + } + + if (i_a < a.length) a_only = a_only.concat(a.slice(i_a)); + if (i_b < b.length) b_only = b_only.concat(b.slice(i_b)); + return { + removed: a_only, + added: b_only + }; +} + +// Convert from wide: { colA: [1,2,3], colB: [4,5,6], ... } +// to long: [ {colA: 1, colB: 4}, {colA: 2, colB: 5}, ... ] +function dataframeToD3(df) { + var names = []; + var length = void 0; + for (var name in df) { + if (df.hasOwnProperty(name)) names.push(name); + if (_typeof(df[name]) !== "object" || typeof df[name].length === "undefined") { + throw new Error("All fields must be arrays"); + } else if (typeof length !== "undefined" && length !== df[name].length) { + throw new Error("All fields must be arrays of the same length"); + } + length = df[name].length; + } + var results = []; + var item = void 0; + for (var row = 0; row < length; row++) { + item = {}; + for (var col = 0; col < names.length; col++) { + item[names[col]] = df[names[col]][row]; + } + results.push(item); + } + return results; +} + +/** + * Keeps track of all event listener additions/removals and lets all active + * listeners be removed with a single operation. + * + * @private + */ + +var SubscriptionTracker = exports.SubscriptionTracker = function () { + function SubscriptionTracker(emitter) { + _classCallCheck(this, SubscriptionTracker); + + this._emitter = emitter; + this._subs = {}; + } + + _createClass(SubscriptionTracker, [{ + key: "on", + value: function on(eventType, listener) { + var sub = this._emitter.on(eventType, listener); + this._subs[sub] = eventType; + return sub; + } + }, { + key: "off", + value: function off(eventType, listener) { + var sub = this._emitter.off(eventType, listener); + if (sub) { + delete this._subs[sub]; + } + return sub; + } + }, { + key: "removeAllListeners", + value: function removeAllListeners() { + var _this = this; + + var current_subs = this._subs; + this._subs = {}; + Object.keys(current_subs).forEach(function (sub) { + _this._emitter.off(current_subs[sub], sub); + }); + } + }]); + + return SubscriptionTracker; +}(); + +},{}],12:[function(require,module,exports){ +(function (global){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _events = require("./events"); + +var _events2 = _interopRequireDefault(_events); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +var Var = function () { + function Var(group, name, /*optional*/value) { + _classCallCheck(this, Var); + + this._group = group; + this._name = name; + this._value = value; + this._events = new _events2.default(); + } + + _createClass(Var, [{ + key: "get", + value: function get() { + return this._value; + } + }, { + key: "set", + value: function set(value, /*optional*/event) { + if (this._value === value) { + // Do nothing; the value hasn't changed + return; + } + var oldValue = this._value; + this._value = value; + // Alert JavaScript listeners that the value has changed + var evt = {}; + if (event && (typeof event === "undefined" ? "undefined" : _typeof(event)) === "object") { + for (var k in event) { + if (event.hasOwnProperty(k)) evt[k] = event[k]; + } + } + evt.oldValue = oldValue; + evt.value = value; + this._events.trigger("change", evt, this); + + // TODO: Make this extensible, to let arbitrary back-ends know that + // something has changed + if (global.Shiny && global.Shiny.onInputChange) { + global.Shiny.onInputChange(".clientValue-" + (this._group.name !== null ? this._group.name + "-" : "") + this._name, typeof value === "undefined" ? null : value); + } + } + }, { + key: "on", + value: function on(eventType, listener) { + return this._events.on(eventType, listener); + } + }, { + key: "off", + value: function off(eventType, listener) { + return this._events.off(eventType, listener); + } + }]); + + return Var; +}(); + +exports.default = Var; + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./events":1}]},{},[5]) +//# sourceMappingURL=crosstalk.js.map diff --git a/_freeze/site_libs/crosstalk-1.2.1/js/crosstalk.js.map b/_freeze/site_libs/crosstalk-1.2.1/js/crosstalk.js.map new file mode 100644 index 00000000..cff94f08 --- /dev/null +++ b/_freeze/site_libs/crosstalk-1.2.1/js/crosstalk.js.map @@ -0,0 +1,37 @@ +{ + "version": 3, + "sources": [ + "node_modules/browser-pack/_prelude.js", + "javascript/src/events.js", + "javascript/src/filter.js", + "javascript/src/filterset.js", + "javascript/src/group.js", + "javascript/src/index.js", + "javascript/src/input.js", + "javascript/src/input_checkboxgroup.js", + "javascript/src/input_selectize.js", + "javascript/src/input_slider.js", + "javascript/src/selection.js", + "javascript/src/util.js", + "javascript/src/var.js" + ], + "names": [], + "mappings": "AAAA;;;;;;;;;;;ICAqB,M;AACnB,oBAAc;AAAA;;AACZ,SAAK,MAAL,GAAc,EAAd;AACA,SAAK,IAAL,GAAY,CAAZ;AACD;;;;uBAEE,S,EAAW,Q,EAAU;AACtB,UAAI,OAAO,KAAK,MAAL,CAAY,SAAZ,CAAX;AACA,UAAI,CAAC,IAAL,EAAW;AACT,eAAO,KAAK,MAAL,CAAY,SAAZ,IAAyB,EAAhC;AACD;AACD,UAAI,MAAM,QAAS,KAAK,IAAL,EAAnB;AACA,WAAK,GAAL,IAAY,QAAZ;AACA,aAAO,GAAP;AACD;;AAED;;;;wBACI,S,EAAW,Q,EAAU;AACvB,UAAI,OAAO,KAAK,MAAL,CAAY,SAAZ,CAAX;AACA,UAAI,OAAO,QAAP,KAAqB,UAAzB,EAAqC;AACnC,aAAK,IAAI,GAAT,IAAgB,IAAhB,EAAsB;AACpB,cAAI,KAAK,cAAL,CAAoB,GAApB,CAAJ,EAA8B;AAC5B,gBAAI,KAAK,GAAL,MAAc,QAAlB,EAA4B;AAC1B,qBAAO,KAAK,GAAL,CAAP;AACA,qBAAO,GAAP;AACD;AACF;AACF;AACD,eAAO,KAAP;AACD,OAVD,MAUO,IAAI,OAAO,QAAP,KAAqB,QAAzB,EAAmC;AACxC,YAAI,QAAQ,KAAK,QAAL,CAAZ,EAA4B;AAC1B,iBAAO,KAAK,QAAL,CAAP;AACA,iBAAO,QAAP;AACD;AACD,eAAO,KAAP;AACD,OANM,MAMA;AACL,cAAM,IAAI,KAAJ,CAAU,8BAAV,CAAN;AACD;AACF;;;4BAEO,S,EAAW,G,EAAK,O,EAAS;AAC/B,UAAI,OAAO,KAAK,MAAL,CAAY,SAAZ,CAAX;AACA,WAAK,IAAI,GAAT,IAAgB,IAAhB,EAAsB;AACpB,YAAI,KAAK,cAAL,CAAoB,GAApB,CAAJ,EAA8B;AAC5B,eAAK,GAAL,EAAU,IAAV,CAAe,OAAf,EAAwB,GAAxB;AACD;AACF;AACF;;;;;;kBA/CkB,M;;;;;;;;;;;;ACArB;;;;AACA;;;;AACA;;;;AACA;;IAAY,I;;;;;;;;AAEZ,SAAS,YAAT,CAAsB,KAAtB,EAA6B;AAC3B,MAAI,QAAQ,MAAM,GAAN,CAAU,WAAV,CAAZ;AACA,MAAI,SAAS,MAAM,GAAN,EAAb;AACA,MAAI,CAAC,MAAL,EAAa;AACX,aAAS,yBAAT;AACA,UAAM,GAAN,CAAU,MAAV;AACD;AACD,SAAO,MAAP;AACD;;AAED,IAAI,KAAK,CAAT;AACA,SAAS,MAAT,GAAkB;AAChB,SAAO,IAAP;AACD;;AAED;;;;;;;;;;;;;;;;;;;;;;;;;IAwBa,Y,WAAA,Y;AACX,wBAAY,KAAZ,EAAmB,SAAnB,EAA8B;AAAA;;AAC5B,SAAK,WAAL,GAAmB,sBAAnB;AACA,SAAK,QAAL,GAAgB,IAAI,KAAK,mBAAT,CAA6B,KAAK,WAAlC,CAAhB;;AAEA;AACA,SAAK,MAAL,GAAc,IAAd;AACA;AACA,SAAK,UAAL,GAAkB,IAAlB;AACA;AACA,SAAK,UAAL,GAAkB,IAAlB;AACA;AACA,SAAK,eAAL,GAAuB,IAAvB;;AAEA,SAAK,UAAL,GAAkB,KAAK,MAAL,CAAY,EAAE,QAAQ,IAAV,EAAZ,EAA8B,SAA9B,CAAlB;;AAEA,SAAK,GAAL,GAAW,WAAW,QAAtB;;AAEA,SAAK,QAAL,CAAc,KAAd;AACD;;AAED;;;;;;;;;;;;;;6BAUS,K,EAAO;AAAA;;AACd;AACA,UAAI,KAAK,MAAL,KAAgB,KAApB,EACE;AACF;AACA,UAAI,CAAC,KAAK,MAAN,IAAgB,CAAC,KAArB,EACE;;AAEF,UAAI,KAAK,UAAT,EAAqB;AACnB,aAAK,UAAL,CAAgB,GAAhB,CAAoB,QAApB,EAA8B,KAAK,eAAnC;AACA,aAAK,KAAL;AACA,aAAK,eAAL,GAAuB,IAAvB;AACA,aAAK,UAAL,GAAkB,IAAlB;AACA,aAAK,UAAL,GAAkB,IAAlB;AACD;;AAED,WAAK,MAAL,GAAc,KAAd;;AAEA,UAAI,KAAJ,EAAW;AACT,gBAAQ,qBAAI,KAAJ,CAAR;AACA,aAAK,UAAL,GAAkB,aAAa,KAAb,CAAlB;AACA,aAAK,UAAL,GAAkB,qBAAI,KAAJ,EAAW,GAAX,CAAe,QAAf,CAAlB;AACA,YAAI,MAAM,KAAK,UAAL,CAAgB,EAAhB,CAAmB,QAAnB,EAA6B,UAAC,CAAD,EAAO;AAC5C,gBAAK,WAAL,CAAiB,OAAjB,CAAyB,QAAzB,EAAmC,CAAnC;AACD,SAFS,CAAV;AAGA,aAAK,eAAL,GAAuB,GAAvB;AACD;AACF;;AAED;;;;;;;;oCAKgB,S,EAAW;AACzB,aAAO,KAAK,MAAL,CAAY,EAAZ,EACL,KAAK,UAAL,GAAkB,KAAK,UAAvB,GAAoC,IAD/B,EAEL,YAAY,SAAZ,GAAwB,IAFnB,CAAP;AAGD;;AAED;;;;;;;4BAIQ;AACN,WAAK,QAAL,CAAc,kBAAd;AACA,WAAK,KAAL;AACA,WAAK,QAAL,CAAc,IAAd;AACD;;AAED;;;;;;;;;;;;0BASM,S,EAAW;AACf,UAAI,CAAC,KAAK,UAAV,EACE;AACF,WAAK,UAAL,CAAgB,KAAhB,CAAsB,KAAK,GAA3B;AACA,WAAK,SAAL,CAAe,SAAf;AACD;;AAED;;;;;;;;;;;;;;;;;;;;wBAiBI,I,EAAM,S,EAAW;AACnB,UAAI,CAAC,KAAK,UAAV,EACE;AACF,WAAK,UAAL,CAAgB,MAAhB,CAAuB,KAAK,GAA5B,EAAiC,IAAjC;AACA,WAAK,SAAL,CAAe,SAAf;AACD;;AAED;;;;;;;;;;AASA;;;;;;;;;;uBAUG,S,EAAW,Q,EAAU;AACtB,aAAO,KAAK,QAAL,CAAc,EAAd,CAAiB,SAAjB,EAA4B,QAA5B,CAAP;AACD;;AAED;;;;;;;;;;;wBAQI,S,EAAW,Q,EAAU;AACvB,aAAO,KAAK,QAAL,CAAc,GAAd,CAAkB,SAAlB,EAA6B,QAA7B,CAAP;AACD;;;8BAES,S,EAAW;AACnB,UAAI,CAAC,KAAK,UAAV,EACE;AACF,WAAK,UAAL,CAAgB,GAAhB,CAAoB,KAAK,UAAL,CAAgB,KAApC,EAA2C,KAAK,eAAL,CAAqB,SAArB,CAA3C;AACD;;AAED;;;;;;;;;;;wBApCmB;AACjB,aAAO,KAAK,UAAL,GAAkB,KAAK,UAAL,CAAgB,KAAlC,GAA0C,IAAjD;AACD;;;;;;AA6CH;;;;;;;;;;;;;;;;;;;ACzNA;;;;AAEA,SAAS,iBAAT,CAA2B,CAA3B,EAA8B,CAA9B,EAAiC;AAC/B,MAAI,MAAM,CAAV,EAAa;AACX,WAAO,CAAP;AACD,GAFD,MAEO,IAAI,IAAI,CAAR,EAAW;AAChB,WAAO,CAAC,CAAR;AACD,GAFM,MAEA,IAAI,IAAI,CAAR,EAAW;AAChB,WAAO,CAAP;AACD;AACF;;AAED;;;;IAGqB,S;AACnB,uBAAc;AAAA;;AACZ,SAAK,KAAL;AACD;;;;4BAEO;AACN;AACA,WAAK,QAAL,GAAgB,EAAhB;AACA;AACA,WAAK,KAAL,GAAa,EAAb;AACA,WAAK,MAAL,GAAc,IAAd;AACA,WAAK,cAAL,GAAsB,CAAtB;AACD;;;2BAMM,Q,EAAU,I,EAAM;AACrB,UAAI,SAAS,IAAb,EAAmB;AACjB,eAAO,KAAK,KAAL,CAAW,CAAX,CAAP,CADiB,CACK;AACtB,aAAK,IAAL,CAAU,iBAAV;AACD;;AAJoB,6BAME,2BAAgB,KAAK,QAAL,CAAc,QAAd,CAAhB,EAAyC,IAAzC,CANF;AAAA,UAMhB,KANgB,oBAMhB,KANgB;AAAA,UAMT,OANS,oBAMT,OANS;;AAOrB,WAAK,QAAL,CAAc,QAAd,IAA0B,IAA1B;;AAEA,WAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,MAAM,MAA1B,EAAkC,GAAlC,EAAuC;AACrC,aAAK,KAAL,CAAW,MAAM,CAAN,CAAX,IAAuB,CAAC,KAAK,KAAL,CAAW,MAAM,CAAN,CAAX,KAAwB,CAAzB,IAA8B,CAArD;AACD;AACD,WAAK,IAAI,KAAI,CAAb,EAAgB,KAAI,QAAQ,MAA5B,EAAoC,IAApC,EAAyC;AACvC,aAAK,KAAL,CAAW,QAAQ,EAAR,CAAX;AACD;;AAED,WAAK,YAAL,CAAkB,IAAlB;AACD;;AAED;;;;;;;;mCAKmC;AAAA,UAAtB,IAAsB,uEAAf,KAAK,QAAU;;AACjC,UAAI,cAAc,OAAO,IAAP,CAAY,KAAK,QAAjB,EAA2B,MAA7C;AACA,UAAI,gBAAgB,CAApB,EAAuB;AACrB,aAAK,MAAL,GAAc,IAAd;AACD,OAFD,MAEO;AACL,aAAK,MAAL,GAAc,EAAd;AACA,aAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,KAAK,MAAzB,EAAiC,GAAjC,EAAsC;AACpC,cAAI,QAAQ,KAAK,KAAL,CAAW,KAAK,CAAL,CAAX,CAAZ;AACA,cAAI,UAAU,WAAd,EAA2B;AACzB,iBAAK,MAAL,CAAY,IAAZ,CAAiB,KAAK,CAAL,CAAjB;AACD;AACF;AACF;AACF;;;0BAEK,Q,EAAU;AACd,UAAI,OAAO,KAAK,QAAL,CAAc,QAAd,CAAP,KAAoC,WAAxC,EAAqD;AACnD;AACD;;AAED,UAAI,OAAO,KAAK,QAAL,CAAc,QAAd,CAAX;AACA,UAAI,CAAC,IAAL,EAAW;AACT,eAAO,EAAP;AACD;;AAED,WAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,KAAK,MAAzB,EAAiC,GAAjC,EAAsC;AACpC,aAAK,KAAL,CAAW,KAAK,CAAL,CAAX;AACD;AACD,aAAO,KAAK,QAAL,CAAc,QAAd,CAAP;;AAEA,WAAK,YAAL;AACD;;;wBA3DW;AACV,aAAO,KAAK,MAAZ;AACD;;;wBA2Dc;AACb,UAAI,UAAU,OAAO,IAAP,CAAY,KAAK,KAAjB,CAAd;AACA,cAAQ,IAAR,CAAa,iBAAb;AACA,aAAO,OAAP;AACD;;;;;;kBA/EkB,S;;;;;;;;;;;;;;kBCRG,K;;AAPxB;;;;;;;;AAEA;AACA;AACA,OAAO,kBAAP,GAA4B,OAAO,kBAAP,IAA6B,EAAzD;AACA,IAAI,SAAS,OAAO,kBAApB;;AAEe,SAAS,KAAT,CAAe,SAAf,EAA0B;AACvC,MAAI,aAAa,OAAO,SAAP,KAAsB,QAAvC,EAAiD;AAC/C,QAAI,CAAC,OAAO,cAAP,CAAsB,SAAtB,CAAL,EAAuC;AACrC,aAAO,SAAP,IAAoB,IAAI,KAAJ,CAAU,SAAV,CAApB;AACD;AACD,WAAO,OAAO,SAAP,CAAP;AACD,GALD,MAKO,IAAI,QAAO,SAAP,yCAAO,SAAP,OAAsB,QAAtB,IAAkC,UAAU,KAA5C,IAAqD,UAAU,GAAnE,EAAwE;AAC7E;AACA,WAAO,SAAP;AACD,GAHM,MAGA,IAAI,MAAM,OAAN,CAAc,SAAd,KACP,UAAU,MAAV,IAAoB,CADb,IAEP,OAAO,UAAU,CAAV,CAAP,KAAyB,QAFtB,EAEgC;AACrC,WAAO,MAAM,UAAU,CAAV,CAAN,CAAP;AACD,GAJM,MAIA;AACL,UAAM,IAAI,KAAJ,CAAU,4BAAV,CAAN;AACD;AACF;;IAEK,K;AACJ,iBAAY,IAAZ,EAAkB;AAAA;;AAChB,SAAK,IAAL,GAAY,IAAZ;AACA,SAAK,KAAL,GAAa,EAAb;AACD;;;;yBAEG,I,EAAM;AACR,UAAI,CAAC,IAAD,IAAS,OAAO,IAAP,KAAiB,QAA9B,EAAwC;AACtC,cAAM,IAAI,KAAJ,CAAU,kBAAV,CAAN;AACD;;AAED,UAAI,CAAC,KAAK,KAAL,CAAW,cAAX,CAA0B,IAA1B,CAAL,EACE,KAAK,KAAL,CAAW,IAAX,IAAmB,kBAAQ,IAAR,EAAc,IAAd,CAAnB;AACF,aAAO,KAAK,KAAL,CAAW,IAAX,CAAP;AACD;;;wBAEG,I,EAAM;AACR,UAAI,CAAC,IAAD,IAAS,OAAO,IAAP,KAAiB,QAA9B,EAAwC;AACtC,cAAM,IAAI,KAAJ,CAAU,kBAAV,CAAN;AACD;;AAED,aAAO,KAAK,KAAL,CAAW,cAAX,CAA0B,IAA1B,CAAP;AACD;;;;;;;;;;;;;;;;AC/CH;;;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAEA,IAAM,eAAe,qBAAM,SAAN,CAArB;;AAEA,SAAS,IAAT,CAAc,IAAd,EAAoB;AAClB,SAAO,aAAa,GAAb,CAAiB,IAAjB,CAAP;AACD;;AAED,SAAS,GAAT,CAAa,IAAb,EAAmB;AACjB,SAAO,aAAa,GAAb,CAAiB,IAAjB,CAAP;AACD;;AAED,IAAI,OAAO,KAAX,EAAkB;AAChB,SAAO,KAAP,CAAa,uBAAb,CAAqC,qBAArC,EAA4D,UAAS,OAAT,EAAkB;AAC5E,QAAI,OAAO,QAAQ,KAAf,KAA0B,QAA9B,EAAwC;AACtC,2BAAM,QAAQ,KAAd,EAAqB,GAArB,CAAyB,QAAQ,IAAjC,EAAuC,GAAvC,CAA2C,QAAQ,KAAnD;AACD,KAFD,MAEO;AACL,WAAK,QAAQ,IAAb,EAAmB,GAAnB,CAAuB,QAAQ,KAA/B;AACD;AACF,GAND;AAOD;;AAED,IAAM,YAAY;AAChB,wBADgB;AAEhB,OAAK,IAFW;AAGhB,OAAK,GAHW;AAIhB,6CAJgB;AAKhB,oCALgB;AAMhB;AANgB,CAAlB;;AASA;;;kBAGe,S;;AACf,OAAO,SAAP,GAAmB,SAAnB;;;;;;;;;;;QCrCgB,Q,GAAA,Q;QAWA,I,GAAA,I;AAfhB,IAAI,IAAI,OAAO,MAAf;;AAEA,IAAI,WAAW,EAAf;;AAEO,SAAS,QAAT,CAAkB,GAAlB,EAAuB;AAC5B,WAAS,IAAI,SAAb,IAA0B,GAA1B;AACA,MAAI,OAAO,QAAP,IAAmB,OAAO,QAAP,CAAgB,UAAhB,KAA+B,UAAtD,EAAkE;AAChE,MAAE,YAAM;AACN;AACD,KAFD;AAGD,GAJD,MAIO,IAAI,OAAO,QAAX,EAAqB;AAC1B,eAAW,IAAX,EAAiB,GAAjB;AACD;AACF;;AAEM,SAAS,IAAT,GAAgB;AACrB,SAAO,IAAP,CAAY,QAAZ,EAAsB,OAAtB,CAA8B,UAAS,SAAT,EAAoB;AAChD,QAAI,UAAU,SAAS,SAAT,CAAd;AACA,MAAE,MAAM,QAAQ,SAAhB,EAA2B,GAA3B,CAA+B,wBAA/B,EAAyD,IAAzD,CAA8D,UAAS,CAAT,EAAY,EAAZ,EAAgB;AAC5E,mBAAa,OAAb,EAAsB,EAAtB;AACD,KAFD;AAGD,GALD;AAMD;;AAED;AACA,SAAS,OAAT,CAAiB,GAAjB,EAAsB;AACpB,SAAO,IAAI,OAAJ,CAAY,uCAAZ,EAAqD,MAArD,CAAP;AACD;;AAED,SAAS,MAAT,CAAgB,EAAhB,EAAoB;AAClB,MAAI,MAAM,EAAE,EAAF,CAAV;AACA,SAAO,IAAP,CAAY,QAAZ,EAAsB,OAAtB,CAA8B,UAAS,SAAT,EAAoB;AAChD,QAAI,IAAI,QAAJ,CAAa,SAAb,KAA2B,CAAC,IAAI,QAAJ,CAAa,uBAAb,CAAhC,EAAuE;AACrE,UAAI,UAAU,SAAS,SAAT,CAAd;AACA,mBAAa,OAAb,EAAsB,EAAtB;AACD;AACF,GALD;AAMD;;AAED,SAAS,YAAT,CAAsB,OAAtB,EAA+B,EAA/B,EAAmC;AACjC,MAAI,SAAS,EAAE,EAAF,EAAM,IAAN,CAAW,+CAA+C,QAAQ,GAAG,EAAX,CAA/C,GAAgE,IAA3E,CAAb;AACA,MAAI,OAAO,KAAK,KAAL,CAAW,OAAO,CAAP,EAAU,SAArB,CAAX;;AAEA,MAAI,WAAW,QAAQ,OAAR,CAAgB,EAAhB,EAAoB,IAApB,CAAf;AACA,IAAE,EAAF,EAAM,IAAN,CAAW,oBAAX,EAAiC,QAAjC;AACA,IAAE,EAAF,EAAM,QAAN,CAAe,uBAAf;AACD;;AAED,IAAI,OAAO,KAAX,EAAkB;AAChB,MAAI,eAAe,IAAI,OAAO,KAAP,CAAa,YAAjB,EAAnB;AACA,MAAI,KAAI,OAAO,MAAf;AACA,KAAE,MAAF,CAAS,YAAT,EAAuB;AACrB,UAAM,cAAS,KAAT,EAAgB;AACpB,aAAO,GAAE,KAAF,EAAS,IAAT,CAAc,kBAAd,CAAP;AACD,KAHoB;AAIrB,gBAAY,oBAAS,EAAT,EAAa;AACvB,UAAI,CAAC,GAAE,EAAF,EAAM,QAAN,CAAe,uBAAf,CAAL,EAA8C;AAC5C,eAAO,EAAP;AACD;AACF,KARoB;AASrB,WAAO,eAAS,EAAT,EAAa;AAClB,aAAO,GAAG,EAAV;AACD,KAXoB;AAYrB,cAAU,kBAAS,EAAT,EAAa,CAEtB,CAdoB;AAerB,cAAU,kBAAS,EAAT,EAAa,KAAb,EAAoB,CAE7B,CAjBoB;AAkBrB,oBAAgB,wBAAS,EAAT,EAAa,IAAb,EAAmB,CAElC,CApBoB;AAqBrB,eAAW,mBAAS,EAAT,EAAa,QAAb,EAAuB;AAChC,SAAE,EAAF,EAAM,IAAN,CAAW,oBAAX,EAAiC,MAAjC;AACD,KAvBoB;AAwBrB,iBAAa,qBAAS,EAAT,EAAa;AACxB,SAAE,EAAF,EAAM,IAAN,CAAW,oBAAX,EAAiC,OAAjC;AACD;AA1BoB,GAAvB;AA4BA,SAAO,KAAP,CAAa,aAAb,CAA2B,QAA3B,CAAoC,YAApC,EAAkD,wBAAlD;AACD;;;;;;;;AChFD;;IAAY,K;;AACZ;;;;AAEA,IAAI,IAAI,OAAO,MAAf;;AAEA,MAAM,QAAN,CAAe;AACb,aAAW,+BADE;;AAGb,WAAS,iBAAS,EAAT,EAAa,IAAb,EAAmB;AAC1B;;;;AAIA,QAAI,WAAW,yBAAiB,KAAK,KAAtB,CAAf;;AAEA,QAAI,sBAAJ;AACA,QAAI,MAAM,EAAE,EAAF,CAAV;AACA,QAAI,EAAJ,CAAO,QAAP,EAAiB,wBAAjB,EAA2C,YAAW;AACpD,UAAI,UAAU,IAAI,IAAJ,CAAS,gCAAT,CAAd;AACA,UAAI,QAAQ,MAAR,KAAmB,CAAvB,EAA0B;AACxB,wBAAgB,IAAhB;AACA,iBAAS,KAAT;AACD,OAHD,MAGO;AACL,YAAI,OAAO,EAAX;AACA,gBAAQ,IAAR,CAAa,YAAW;AACtB,eAAK,GAAL,CAAS,KAAK,KAAd,EAAqB,OAArB,CAA6B,UAAS,GAAT,EAAc;AACzC,iBAAK,GAAL,IAAY,IAAZ;AACD,WAFD;AAGD,SAJD;AAKA,YAAI,WAAW,OAAO,IAAP,CAAY,IAAZ,CAAf;AACA,iBAAS,IAAT;AACA,wBAAgB,QAAhB;AACA,iBAAS,GAAT,CAAa,QAAb;AACD;AACF,KAjBD;;AAmBA,WAAO;AACL,eAAS,mBAAW;AAClB,iBAAS,KAAT;AACD,OAHI;AAIL,cAAQ,kBAAW;AACjB,YAAI,aAAJ,EACE,SAAS,GAAT,CAAa,aAAb;AACH;AAPI,KAAP;AASD;AAxCY,CAAf;;;;;;;;ACLA;;IAAY,K;;AACZ;;IAAY,I;;AACZ;;;;AAEA,IAAI,IAAI,OAAO,MAAf;;AAEA,MAAM,QAAN,CAAe;AACb,aAAW,wBADE;;AAGb,WAAS,iBAAS,EAAT,EAAa,IAAb,EAAmB;AAC1B;;;;;;AAMA,QAAI,QAAQ,CAAC,EAAC,OAAO,EAAR,EAAY,OAAO,OAAnB,EAAD,CAAZ;AACA,QAAI,QAAQ,KAAK,aAAL,CAAmB,KAAK,KAAxB,CAAZ;AACA,QAAI,OAAO;AACT,eAAS,MAAM,MAAN,CAAa,KAAb,CADA;AAET,kBAAY,OAFH;AAGT,kBAAY,OAHH;AAIT,mBAAa;AAJJ,KAAX;;AAOA,QAAI,SAAS,EAAE,EAAF,EAAM,IAAN,CAAW,QAAX,EAAqB,CAArB,CAAb;;AAEA,QAAI,YAAY,EAAE,MAAF,EAAU,SAAV,CAAoB,IAApB,EAA0B,CAA1B,EAA6B,SAA7C;;AAEA,QAAI,WAAW,yBAAiB,KAAK,KAAtB,CAAf;;AAEA,QAAI,sBAAJ;AACA,cAAU,EAAV,CAAa,QAAb,EAAuB,YAAW;AAChC,UAAI,UAAU,KAAV,CAAgB,MAAhB,KAA2B,CAA/B,EAAkC;AAChC,wBAAgB,IAAhB;AACA,iBAAS,KAAT;AACD,OAHD,MAGO;AACL,YAAI,OAAO,EAAX;AACA,kBAAU,KAAV,CAAgB,OAAhB,CAAwB,UAAS,KAAT,EAAgB;AACtC,eAAK,GAAL,CAAS,KAAT,EAAgB,OAAhB,CAAwB,UAAS,GAAT,EAAc;AACpC,iBAAK,GAAL,IAAY,IAAZ;AACD,WAFD;AAGD,SAJD;AAKA,YAAI,WAAW,OAAO,IAAP,CAAY,IAAZ,CAAf;AACA,iBAAS,IAAT;AACA,wBAAgB,QAAhB;AACA,iBAAS,GAAT,CAAa,QAAb;AACD;AACF,KAhBD;;AAkBA,WAAO;AACL,eAAS,mBAAW;AAClB,iBAAS,KAAT;AACD,OAHI;AAIL,cAAQ,kBAAW;AACjB,YAAI,aAAJ,EACE,SAAS,GAAT,CAAa,aAAb;AACH;AAPI,KAAP;AASD;AArDY,CAAf;;;;;;;;;;ACNA;;IAAY,K;;AACZ;;;;AAEA,IAAI,IAAI,OAAO,MAAf;AACA,IAAI,WAAW,OAAO,QAAtB;;AAEA,MAAM,QAAN,CAAe;AACb,aAAW,wBADE;;AAGb,WAAS,iBAAS,EAAT,EAAa,IAAb,EAAmB;AAC1B;;;;AAIA,QAAI,WAAW,yBAAiB,KAAK,KAAtB,CAAf;;AAEA,QAAI,OAAO,EAAX;AACA,QAAI,MAAM,EAAE,EAAF,EAAM,IAAN,CAAW,OAAX,CAAV;AACA,QAAI,WAAW,IAAI,IAAJ,CAAS,WAAT,CAAf;AACA,QAAI,aAAa,IAAI,IAAJ,CAAS,aAAT,CAAjB;AACA,QAAI,QAAQ,IAAI,IAAJ,CAAS,OAAT,CAAZ;AACA,QAAI,sBAAJ;;AAEA;AACA,QAAI,aAAa,MAAjB,EAAyB;AACvB,sBAAgB,SAAS,GAAT,EAAhB;AACA,WAAK,QAAL,GAAgB,UAAS,GAAT,EAAc;AAC5B,eAAO,cAAc,UAAd,EAA0B,IAAI,IAAJ,CAAS,GAAT,CAA1B,CAAP;AACD,OAFD;AAID,KAND,MAMO,IAAI,aAAa,UAAjB,EAA6B;AAClC,UAAI,WAAW,IAAI,IAAJ,CAAS,UAAT,CAAf;AACA,UAAI,QAAJ,EACE,gBAAgB,SAAS,QAAT,CAAkB,QAAlB,CAAhB,CADF,KAGE,gBAAgB,QAAhB;;AAEF,WAAK,QAAL,GAAgB,UAAS,GAAT,EAAc;AAC5B,eAAO,cAAc,UAAd,EAA0B,IAAI,IAAJ,CAAS,GAAT,CAA1B,CAAP;AACD,OAFD;AAGD,KAVM,MAUA,IAAI,aAAa,QAAjB,EAA2B;AAChC,UAAI,OAAO,KAAP,KAAiB,WAArB,EACE,KAAK,QAAL,GAAgB,UAAS,GAAT,EAAc;AAC5B,YAAI,SAAS,KAAK,GAAL,CAAS,EAAT,EAAa,KAAb,CAAb;AACA,eAAO,KAAK,KAAL,CAAW,MAAM,MAAjB,IAA2B,MAAlC;AACD,OAHD;AAIH;;AAED,QAAI,cAAJ,CAAmB,IAAnB;;AAEA,aAAS,QAAT,GAAoB;AAClB,UAAI,SAAS,IAAI,IAAJ,CAAS,gBAAT,EAA2B,MAAxC;;AAEA;AACA,UAAI,gBAAJ;AACA,UAAI,WAAW,IAAI,IAAJ,CAAS,WAAT,CAAf;AACA,UAAI,aAAa,MAAjB,EAAyB;AACvB,kBAAU,iBAAS,GAAT,EAAc;AACtB,iBAAO,cAAc,IAAI,IAAJ,CAAS,CAAC,GAAV,CAAd,CAAP;AACD,SAFD;AAGD,OAJD,MAIO,IAAI,aAAa,UAAjB,EAA6B;AAClC,kBAAU,iBAAS,GAAT,EAAc;AACtB;AACA,iBAAO,CAAC,GAAD,GAAO,IAAd;AACD,SAHD;AAID,OALM,MAKA;AACL,kBAAU,iBAAS,GAAT,EAAc;AAAE,iBAAO,CAAC,GAAR;AAAc,SAAxC;AACD;;AAED,UAAI,IAAI,IAAJ,CAAS,gBAAT,EAA2B,OAA3B,CAAmC,IAAnC,KAA4C,QAAhD,EAA0D;AACxD,eAAO,CAAC,QAAQ,OAAO,IAAf,CAAD,EAAuB,QAAQ,OAAO,EAAf,CAAvB,CAAP;AACD,OAFD,MAEO;AACL,eAAO,QAAQ,OAAO,IAAf,CAAP;AACD;AACF;;AAED,QAAI,gBAAgB,IAApB;;AAEA,QAAI,EAAJ,CAAO,6BAAP,EAAsC,UAAS,KAAT,EAAgB;AACpD,UAAI,CAAC,IAAI,IAAJ,CAAS,UAAT,CAAD,IAAyB,CAAC,IAAI,IAAJ,CAAS,WAAT,CAA9B,EAAqD;AAAA,wBAClC,UADkC;AAAA;AAAA,YAC9C,IAD8C;AAAA,YACxC,EADwC;;AAEnD,YAAI,OAAO,EAAX;AACA,aAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,KAAK,MAAL,CAAY,MAAhC,EAAwC,GAAxC,EAA6C;AAC3C,cAAI,MAAM,KAAK,MAAL,CAAY,CAAZ,CAAV;AACA,cAAI,OAAO,IAAP,IAAe,OAAO,EAA1B,EAA8B;AAC5B,iBAAK,IAAL,CAAU,KAAK,IAAL,CAAU,CAAV,CAAV;AACD;AACF;AACD,aAAK,IAAL;AACA,iBAAS,GAAT,CAAa,IAAb;AACA,wBAAgB,IAAhB;AACD;AACF,KAdD;;AAiBA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,WAAO;AACL,eAAS,mBAAW;AAClB,iBAAS,KAAT;AACD,OAHI;AAIL,cAAQ,kBAAW;AACjB,YAAI,aAAJ,EACE,SAAS,GAAT,CAAa,aAAb;AACH;AAPI,KAAP;AASD;AApHY,CAAf;;AAwHA;AACA,SAAS,QAAT,CAAkB,CAAlB,EAAqB,MAArB,EAA6B;AAC3B,MAAI,MAAM,EAAE,QAAF,EAAV;AACA,SAAO,IAAI,MAAJ,GAAa,MAApB;AACE,UAAM,MAAM,GAAZ;AADF,GAEA,OAAO,GAAP;AACD;;AAED;AACA;AACA,SAAS,aAAT,CAAuB,IAAvB,EAA6B;AAC3B,MAAI,gBAAgB,IAApB,EAA0B;AACxB,WAAO,KAAK,cAAL,KAAwB,GAAxB,GACA,SAAS,KAAK,WAAL,KAAmB,CAA5B,EAA+B,CAA/B,CADA,GACoC,GADpC,GAEA,SAAS,KAAK,UAAL,EAAT,EAA4B,CAA5B,CAFP;AAID,GALD,MAKO;AACL,WAAO,IAAP;AACD;AACF;;;;;;;;;;;;;;ACjJD;;;;AACA;;;;AACA;;IAAY,I;;;;;;;;AAEZ;;;;;;;;;;;;;;;;IAgBa,e,WAAA,e;AAEX,6BAA4C;AAAA,QAAhC,KAAgC,uEAAxB,IAAwB;AAAA,QAAlB,SAAkB,uEAAN,IAAM;;AAAA;;AAC1C,SAAK,WAAL,GAAmB,sBAAnB;AACA,SAAK,QAAL,GAAgB,IAAI,KAAK,mBAAT,CAA6B,KAAK,WAAlC,CAAhB;;AAEA;AACA,SAAK,MAAL,GAAc,IAAd;AACA;AACA,SAAK,IAAL,GAAY,IAAZ;AACA;AACA,SAAK,eAAL,GAAuB,IAAvB;;AAEA,SAAK,UAAL,GAAkB,KAAK,MAAL,CAAY,EAAE,QAAQ,IAAV,EAAZ,EAA8B,SAA9B,CAAlB;;AAEA,SAAK,QAAL,CAAc,KAAd;AACD;;AAED;;;;;;;;;;;;;;;;;6BAaS,K,EAAO;AAAA;;AACd;AACA,UAAI,KAAK,MAAL,KAAgB,KAApB,EACE;AACF;AACA,UAAI,CAAC,KAAK,MAAN,IAAgB,CAAC,KAArB,EACE;;AAEF,UAAI,KAAK,IAAT,EAAe;AACb,aAAK,IAAL,CAAU,GAAV,CAAc,QAAd,EAAwB,KAAK,eAA7B;AACA,aAAK,IAAL,GAAY,IAAZ;AACA,aAAK,eAAL,GAAuB,IAAvB;AACD;;AAED,WAAK,MAAL,GAAc,KAAd;;AAEA,UAAI,KAAJ,EAAW;AACT,aAAK,IAAL,GAAY,qBAAI,KAAJ,EAAW,GAAX,CAAe,WAAf,CAAZ;AACA,YAAI,MAAM,KAAK,IAAL,CAAU,EAAV,CAAa,QAAb,EAAuB,UAAC,CAAD,EAAO;AACtC,gBAAK,WAAL,CAAiB,OAAjB,CAAyB,QAAzB,EAAmC,CAAnC;AACD,SAFS,CAAV;AAGA,aAAK,eAAL,GAAuB,GAAvB;AACD;AACF;;AAED;;;;;;;;;;;;;;;AAcA;;;;;oCAKgB,S,EAAW;AACzB;AACA,aAAO,KAAK,MAAL,CAAY,EAAZ,EACL,KAAK,UAAL,GAAkB,KAAK,UAAvB,GAAoC,IAD/B,EAEL,YAAY,SAAZ,GAAwB,IAFnB,CAAP;AAGD;;AAED;;;;;;;;;;;;;;;wBAYI,Y,EAAc,S,EAAW;AAC3B,UAAI,KAAK,IAAT,EACE,KAAK,IAAL,CAAU,GAAV,CAAc,YAAd,EAA4B,KAAK,eAAL,CAAqB,SAArB,CAA5B;AACH;;AAED;;;;;;;;;;;;;0BAUM,S,EAAW;AACf,UAAI,KAAK,IAAT,EACE,KAAK,GAAL,CAAS,KAAK,CAAd,EAAiB,KAAK,eAAL,CAAqB,SAArB,CAAjB;AACH;;AAED;;;;;;;;;;;;;uBAUG,S,EAAW,Q,EAAU;AACtB,aAAO,KAAK,QAAL,CAAc,EAAd,CAAiB,SAAjB,EAA4B,QAA5B,CAAP;AACD;;AAED;;;;;;;;;;;wBAQI,S,EAAW,Q,EAAU;AACvB,aAAO,KAAK,QAAL,CAAc,GAAd,CAAkB,SAAlB,EAA6B,QAA7B,CAAP;AACD;;AAED;;;;;;;;4BAKQ;AACN,WAAK,QAAL,CAAc,kBAAd;AACA,WAAK,QAAL,CAAc,IAAd;AACD;;;wBAlFW;AACV,aAAO,KAAK,IAAL,GAAY,KAAK,IAAL,CAAU,GAAV,EAAZ,GAA8B,IAArC;AACD;;;;;;AAmFH;;;;;;;;;AASA;;;;;;;;;;;;;;;;;;;;;QCpLgB,M,GAAA,M;QAeA,W,GAAA,W;QAQA,e,GAAA,e;QAoCA,a,GAAA,a;;;;AA3DT,SAAS,MAAT,CAAgB,MAAhB,EAAoC;AAAA,oCAAT,OAAS;AAAT,WAAS;AAAA;;AACzC,OAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,QAAQ,MAA5B,EAAoC,GAApC,EAAyC;AACvC,QAAI,MAAM,QAAQ,CAAR,CAAV;AACA,QAAI,OAAO,GAAP,KAAgB,WAAhB,IAA+B,QAAQ,IAA3C,EACE;;AAEF,SAAK,IAAI,GAAT,IAAgB,GAAhB,EAAqB;AACnB,UAAI,IAAI,cAAJ,CAAmB,GAAnB,CAAJ,EAA6B;AAC3B,eAAO,GAAP,IAAc,IAAI,GAAJ,CAAd;AACD;AACF;AACF;AACD,SAAO,MAAP;AACD;;AAEM,SAAS,WAAT,CAAqB,IAArB,EAA2B;AAChC,OAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,KAAK,MAAzB,EAAiC,GAAjC,EAAsC;AACpC,QAAI,KAAK,CAAL,KAAW,KAAK,IAAE,CAAP,CAAf,EAA0B;AACxB,YAAM,IAAI,KAAJ,CAAU,0CAAV,CAAN;AACD;AACF;AACF;;AAEM,SAAS,eAAT,CAAyB,CAAzB,EAA4B,CAA5B,EAA+B;AACpC,MAAI,MAAM,CAAV;AACA,MAAI,MAAM,CAAV;;AAEA,MAAI,CAAC,CAAL,EAAQ,IAAI,EAAJ;AACR,MAAI,CAAC,CAAL,EAAQ,IAAI,EAAJ;;AAER,MAAI,SAAS,EAAb;AACA,MAAI,SAAS,EAAb;;AAEA,cAAY,CAAZ;AACA,cAAY,CAAZ;;AAEA,SAAO,MAAM,EAAE,MAAR,IAAkB,MAAM,EAAE,MAAjC,EAAyC;AACvC,QAAI,EAAE,GAAF,MAAW,EAAE,GAAF,CAAf,EAAuB;AACrB;AACA;AACD,KAHD,MAGO,IAAI,EAAE,GAAF,IAAS,EAAE,GAAF,CAAb,EAAqB;AAC1B,aAAO,IAAP,CAAY,EAAE,KAAF,CAAZ;AACD,KAFM,MAEA;AACL,aAAO,IAAP,CAAY,EAAE,KAAF,CAAZ;AACD;AACF;;AAED,MAAI,MAAM,EAAE,MAAZ,EACE,SAAS,OAAO,MAAP,CAAc,EAAE,KAAF,CAAQ,GAAR,CAAd,CAAT;AACF,MAAI,MAAM,EAAE,MAAZ,EACE,SAAS,OAAO,MAAP,CAAc,EAAE,KAAF,CAAQ,GAAR,CAAd,CAAT;AACF,SAAO;AACL,aAAS,MADJ;AAEL,WAAO;AAFF,GAAP;AAID;;AAED;AACA;AACO,SAAS,aAAT,CAAuB,EAAvB,EAA2B;AAChC,MAAI,QAAQ,EAAZ;AACA,MAAI,eAAJ;AACA,OAAK,IAAI,IAAT,IAAiB,EAAjB,EAAqB;AACnB,QAAI,GAAG,cAAH,CAAkB,IAAlB,CAAJ,EACE,MAAM,IAAN,CAAW,IAAX;AACF,QAAI,QAAO,GAAG,IAAH,CAAP,MAAqB,QAArB,IAAiC,OAAO,GAAG,IAAH,EAAS,MAAhB,KAA4B,WAAjE,EAA8E;AAC5E,YAAM,IAAI,KAAJ,CAAU,2BAAV,CAAN;AACD,KAFD,MAEO,IAAI,OAAO,MAAP,KAAmB,WAAnB,IAAkC,WAAW,GAAG,IAAH,EAAS,MAA1D,EAAkE;AACvE,YAAM,IAAI,KAAJ,CAAU,8CAAV,CAAN;AACD;AACD,aAAS,GAAG,IAAH,EAAS,MAAlB;AACD;AACD,MAAI,UAAU,EAAd;AACA,MAAI,aAAJ;AACA,OAAK,IAAI,MAAM,CAAf,EAAkB,MAAM,MAAxB,EAAgC,KAAhC,EAAuC;AACrC,WAAO,EAAP;AACA,SAAK,IAAI,MAAM,CAAf,EAAkB,MAAM,MAAM,MAA9B,EAAsC,KAAtC,EAA6C;AAC3C,WAAK,MAAM,GAAN,CAAL,IAAmB,GAAG,MAAM,GAAN,CAAH,EAAe,GAAf,CAAnB;AACD;AACD,YAAQ,IAAR,CAAa,IAAb;AACD;AACD,SAAO,OAAP;AACD;;AAED;;;;;;;IAMa,mB,WAAA,mB;AACX,+BAAY,OAAZ,EAAqB;AAAA;;AACnB,SAAK,QAAL,GAAgB,OAAhB;AACA,SAAK,KAAL,GAAa,EAAb;AACD;;;;uBAEE,S,EAAW,Q,EAAU;AACtB,UAAI,MAAM,KAAK,QAAL,CAAc,EAAd,CAAiB,SAAjB,EAA4B,QAA5B,CAAV;AACA,WAAK,KAAL,CAAW,GAAX,IAAkB,SAAlB;AACA,aAAO,GAAP;AACD;;;wBAEG,S,EAAW,Q,EAAU;AACvB,UAAI,MAAM,KAAK,QAAL,CAAc,GAAd,CAAkB,SAAlB,EAA6B,QAA7B,CAAV;AACA,UAAI,GAAJ,EAAS;AACP,eAAO,KAAK,KAAL,CAAW,GAAX,CAAP;AACD;AACD,aAAO,GAAP;AACD;;;yCAEoB;AAAA;;AACnB,UAAI,eAAe,KAAK,KAAxB;AACA,WAAK,KAAL,GAAa,EAAb;AACA,aAAO,IAAP,CAAY,YAAZ,EAA0B,OAA1B,CAAkC,UAAC,GAAD,EAAS;AACzC,cAAK,QAAL,CAAc,GAAd,CAAkB,aAAa,GAAb,CAAlB,EAAqC,GAArC;AACD,OAFD;AAGD;;;;;;;;;;;;;;;;;;ACpHH;;;;;;;;IAEqB,G;AACnB,eAAY,KAAZ,EAAmB,IAAnB,EAAyB,YAAa,KAAtC,EAA6C;AAAA;;AAC3C,SAAK,MAAL,GAAc,KAAd;AACA,SAAK,KAAL,GAAa,IAAb;AACA,SAAK,MAAL,GAAc,KAAd;AACA,SAAK,OAAL,GAAe,sBAAf;AACD;;;;0BAEK;AACJ,aAAO,KAAK,MAAZ;AACD;;;wBAEG,K,EAAO,YAAa,K,EAAO;AAC7B,UAAI,KAAK,MAAL,KAAgB,KAApB,EAA2B;AACzB;AACA;AACD;AACD,UAAI,WAAW,KAAK,MAApB;AACA,WAAK,MAAL,GAAc,KAAd;AACA;AACA,UAAI,MAAM,EAAV;AACA,UAAI,SAAS,QAAO,KAAP,yCAAO,KAAP,OAAkB,QAA/B,EAAyC;AACvC,aAAK,IAAI,CAAT,IAAc,KAAd,EAAqB;AACnB,cAAI,MAAM,cAAN,CAAqB,CAArB,CAAJ,EACE,IAAI,CAAJ,IAAS,MAAM,CAAN,CAAT;AACH;AACF;AACD,UAAI,QAAJ,GAAe,QAAf;AACA,UAAI,KAAJ,GAAY,KAAZ;AACA,WAAK,OAAL,CAAa,OAAb,CAAqB,QAArB,EAA+B,GAA/B,EAAoC,IAApC;;AAEA;AACA;AACA,UAAI,OAAO,KAAP,IAAgB,OAAO,KAAP,CAAa,aAAjC,EAAgD;AAC9C,eAAO,KAAP,CAAa,aAAb,CACE,mBACG,KAAK,MAAL,CAAY,IAAZ,KAAqB,IAArB,GAA4B,KAAK,MAAL,CAAY,IAAZ,GAAmB,GAA/C,GAAqD,EADxD,IAEE,KAAK,KAHT,EAIE,OAAO,KAAP,KAAkB,WAAlB,GAAgC,IAAhC,GAAuC,KAJzC;AAMD;AACF;;;uBAEE,S,EAAW,Q,EAAU;AACtB,aAAO,KAAK,OAAL,CAAa,EAAb,CAAgB,SAAhB,EAA2B,QAA3B,CAAP;AACD;;;wBAEG,S,EAAW,Q,EAAU;AACvB,aAAO,KAAK,OAAL,CAAa,GAAb,CAAiB,SAAjB,EAA4B,QAA5B,CAAP;AACD;;;;;;kBAjDkB,G", + "file": "generated.js", + "sourceRoot": "", + "sourcesContent": [ + "(function(){function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require==\"function\"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error(\"Cannot find module '\"+o+\"'\");throw f.code=\"MODULE_NOT_FOUND\",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require==\"function\"&&require;for(var o=0;o {\n this._eventRelay.trigger(\"change\", e, this);\n });\n this._varOnChangeSub = sub;\n }\n }\n\n /**\n * Combine the given `extraInfo` (if any) with the handle's default\n * `_extraInfo` (if any).\n * @private\n */\n _mergeExtraInfo(extraInfo) {\n return util.extend({},\n this._extraInfo ? this._extraInfo : null,\n extraInfo ? extraInfo : null);\n }\n\n /**\n * Close the handle. This clears this handle's contribution to the filter set,\n * and unsubscribes all event listeners.\n */\n close() {\n this._emitter.removeAllListeners();\n this.clear();\n this.setGroup(null);\n }\n\n /**\n * Clear this handle's contribution to the filter set.\n *\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `FilterHandle` constructor).\n * \n * @fires FilterHandle#change\n */\n clear(extraInfo) {\n if (!this._filterSet)\n return;\n this._filterSet.clear(this._id);\n this._onChange(extraInfo);\n }\n\n /**\n * Set this handle's contribution to the filter set. This array should consist\n * of the keys of the rows that _should_ be displayed; any keys that are not\n * present in the array will be considered _filtered out_. Note that multiple\n * `FilterHandle` instances in the group may each contribute an array of keys,\n * and only those keys that appear in _all_ of the arrays make it through the\n * filter.\n *\n * @param {string[]} keys - Empty array, or array of keys. To clear the\n * filter, don't pass an empty array; instead, use the\n * {@link FilterHandle#clear} method.\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `FilterHandle` constructor).\n * \n * @fires FilterHandle#change\n */\n set(keys, extraInfo) {\n if (!this._filterSet)\n return;\n this._filterSet.update(this._id, keys);\n this._onChange(extraInfo);\n }\n\n /**\n * @return {string[]|null} - Either: 1) an array of keys that made it through\n * all of the `FilterHandle` instances, or, 2) `null`, which means no filter\n * is being applied (all data should be displayed).\n */\n get filteredKeys() {\n return this._filterSet ? this._filterSet.value : null;\n }\n\n /**\n * Subscribe to events on this `FilterHandle`.\n *\n * @param {string} eventType - Indicates the type of events to listen to.\n * Currently, only `\"change\"` is supported.\n * @param {FilterHandle~listener} listener - The callback function that\n * will be invoked when the event occurs.\n * @return {string} - A token to pass to {@link FilterHandle#off} to cancel\n * this subscription.\n */\n on(eventType, listener) {\n return this._emitter.on(eventType, listener);\n }\n\n /**\n * Cancel event subscriptions created by {@link FilterHandle#on}.\n *\n * @param {string} eventType - The type of event to unsubscribe.\n * @param {string|FilterHandle~listener} listener - Either the callback\n * function previously passed into {@link FilterHandle#on}, or the\n * string that was returned from {@link FilterHandle#on}.\n */\n off(eventType, listener) {\n return this._emitter.off(eventType, listener);\n }\n\n _onChange(extraInfo) {\n if (!this._filterSet)\n return;\n this._filterVar.set(this._filterSet.value, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * @callback FilterHandle~listener\n * @param {Object} event - An object containing details of the event. For\n * `\"change\"` events, this includes the properties `value` (the new\n * value of the filter set, or `null` if no filter set is active),\n * `oldValue` (the previous value of the filter set), and `sender` (the\n * `FilterHandle` instance that made the change).\n */\n\n}\n\n/**\n * @event FilterHandle#change\n * @type {object}\n * @property {object} value - The new value of the filter set, or `null`\n * if no filter set is active.\n * @property {object} oldValue - The previous value of the filter set.\n * @property {FilterHandle} sender - The `FilterHandle` instance that\n * changed the value.\n */\n", + "import { diffSortedLists } from \"./util\";\n\nfunction naturalComparator(a, b) {\n if (a === b) {\n return 0;\n } else if (a < b) {\n return -1;\n } else if (a > b) {\n return 1;\n }\n}\n\n/**\n * @private\n */\nexport default class FilterSet {\n constructor() {\n this.reset();\n }\n\n reset() {\n // Key: handle ID, Value: array of selected keys, or null\n this._handles = {};\n // Key: key string, Value: count of handles that include it\n this._keys = {};\n this._value = null;\n this._activeHandles = 0;\n }\n\n get value() {\n return this._value;\n }\n\n update(handleId, keys) {\n if (keys !== null) {\n keys = keys.slice(0); // clone before sorting\n keys.sort(naturalComparator);\n }\n\n let {added, removed} = diffSortedLists(this._handles[handleId], keys);\n this._handles[handleId] = keys;\n\n for (let i = 0; i < added.length; i++) {\n this._keys[added[i]] = (this._keys[added[i]] || 0) + 1;\n }\n for (let i = 0; i < removed.length; i++) {\n this._keys[removed[i]]--;\n }\n\n this._updateValue(keys);\n }\n\n /**\n * @param {string[]} keys Sorted array of strings that indicate\n * a superset of possible keys.\n * @private\n */\n _updateValue(keys = this._allKeys) {\n let handleCount = Object.keys(this._handles).length;\n if (handleCount === 0) {\n this._value = null;\n } else {\n this._value = [];\n for (let i = 0; i < keys.length; i++) {\n let count = this._keys[keys[i]];\n if (count === handleCount) {\n this._value.push(keys[i]);\n }\n }\n }\n }\n\n clear(handleId) {\n if (typeof(this._handles[handleId]) === \"undefined\") {\n return;\n }\n\n let keys = this._handles[handleId];\n if (!keys) {\n keys = [];\n }\n\n for (let i = 0; i < keys.length; i++) {\n this._keys[keys[i]]--;\n }\n delete this._handles[handleId];\n\n this._updateValue();\n }\n\n get _allKeys() {\n let allKeys = Object.keys(this._keys);\n allKeys.sort(naturalComparator);\n return allKeys;\n }\n}\n", + "import Var from \"./var\";\n\n// Use a global so that multiple copies of crosstalk.js can be loaded and still\n// have groups behave as singletons across all copies.\nglobal.__crosstalk_groups = global.__crosstalk_groups || {};\nlet groups = global.__crosstalk_groups;\n\nexport default function group(groupName) {\n if (groupName && typeof(groupName) === \"string\") {\n if (!groups.hasOwnProperty(groupName)) {\n groups[groupName] = new Group(groupName);\n }\n return groups[groupName];\n } else if (typeof(groupName) === \"object\" && groupName._vars && groupName.var) {\n // Appears to already be a group object\n return groupName;\n } else if (Array.isArray(groupName) &&\n groupName.length == 1 &&\n typeof(groupName[0]) === \"string\") {\n return group(groupName[0]);\n } else {\n throw new Error(\"Invalid groupName argument\");\n }\n}\n\nclass Group {\n constructor(name) {\n this.name = name;\n this._vars = {};\n }\n\n var(name) {\n if (!name || typeof(name) !== \"string\") {\n throw new Error(\"Invalid var name\");\n }\n\n if (!this._vars.hasOwnProperty(name))\n this._vars[name] = new Var(this, name);\n return this._vars[name];\n }\n\n has(name) {\n if (!name || typeof(name) !== \"string\") {\n throw new Error(\"Invalid var name\");\n }\n\n return this._vars.hasOwnProperty(name);\n }\n}\n", + "import group from \"./group\";\nimport { SelectionHandle } from \"./selection\";\nimport { FilterHandle } from \"./filter\";\nimport { bind } from \"./input\";\nimport \"./input_selectize\";\nimport \"./input_checkboxgroup\";\nimport \"./input_slider\";\n\nconst defaultGroup = group(\"default\");\n\nfunction var_(name) {\n return defaultGroup.var(name);\n}\n\nfunction has(name) {\n return defaultGroup.has(name);\n}\n\nif (global.Shiny) {\n global.Shiny.addCustomMessageHandler(\"update-client-value\", function(message) {\n if (typeof(message.group) === \"string\") {\n group(message.group).var(message.name).set(message.value);\n } else {\n var_(message.name).set(message.value);\n }\n });\n}\n\nconst crosstalk = {\n group: group,\n var: var_,\n has: has,\n SelectionHandle: SelectionHandle,\n FilterHandle: FilterHandle,\n bind: bind\n};\n\n/**\n * @namespace crosstalk\n */\nexport default crosstalk;\nglobal.crosstalk = crosstalk;\n", + "let $ = global.jQuery;\n\nlet bindings = {};\n\nexport function register(reg) {\n bindings[reg.className] = reg;\n if (global.document && global.document.readyState !== \"complete\") {\n $(() => {\n bind();\n });\n } else if (global.document) {\n setTimeout(bind, 100);\n }\n}\n\nexport function bind() {\n Object.keys(bindings).forEach(function(className) {\n let binding = bindings[className];\n $(\".\" + binding.className).not(\".crosstalk-input-bound\").each(function(i, el) {\n bindInstance(binding, el);\n });\n });\n}\n\n// Escape jQuery identifier\nfunction $escape(val) {\n return val.replace(/([!\"#$%&'()*+,./:;<=>?@[\\\\\\]^`{|}~])/g, \"\\\\$1\");\n}\n\nfunction bindEl(el) {\n let $el = $(el);\n Object.keys(bindings).forEach(function(className) {\n if ($el.hasClass(className) && !$el.hasClass(\"crosstalk-input-bound\")) {\n let binding = bindings[className];\n bindInstance(binding, el);\n }\n });\n}\n\nfunction bindInstance(binding, el) {\n let jsonEl = $(el).find(\"script[type='application/json'][data-for='\" + $escape(el.id) + \"']\");\n let data = JSON.parse(jsonEl[0].innerText);\n\n let instance = binding.factory(el, data);\n $(el).data(\"crosstalk-instance\", instance);\n $(el).addClass(\"crosstalk-input-bound\");\n}\n\nif (global.Shiny) {\n let inputBinding = new global.Shiny.InputBinding();\n let $ = global.jQuery;\n $.extend(inputBinding, {\n find: function(scope) {\n return $(scope).find(\".crosstalk-input\");\n },\n initialize: function(el) {\n if (!$(el).hasClass(\"crosstalk-input-bound\")) {\n bindEl(el);\n }\n },\n getId: function(el) {\n return el.id;\n },\n getValue: function(el) {\n\n },\n setValue: function(el, value) {\n\n },\n receiveMessage: function(el, data) {\n\n },\n subscribe: function(el, callback) {\n $(el).data(\"crosstalk-instance\").resume();\n },\n unsubscribe: function(el) {\n $(el).data(\"crosstalk-instance\").suspend();\n }\n });\n global.Shiny.inputBindings.register(inputBinding, \"crosstalk.inputBinding\");\n}\n", + "import * as input from \"./input\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\n\ninput.register({\n className: \"crosstalk-input-checkboxgroup\",\n\n factory: function(el, data) {\n /*\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n let ctHandle = new FilterHandle(data.group);\n\n let lastKnownKeys;\n let $el = $(el);\n $el.on(\"change\", \"input[type='checkbox']\", function() {\n let checked = $el.find(\"input[type='checkbox']:checked\");\n if (checked.length === 0) {\n lastKnownKeys = null;\n ctHandle.clear();\n } else {\n let keys = {};\n checked.each(function() {\n data.map[this.value].forEach(function(key) {\n keys[key] = true;\n });\n });\n let keyArray = Object.keys(keys);\n keyArray.sort();\n lastKnownKeys = keyArray;\n ctHandle.set(keyArray);\n }\n });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n", + "import * as input from \"./input\";\nimport * as util from \"./util\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\n\ninput.register({\n className: \"crosstalk-input-select\",\n\n factory: function(el, data) {\n /*\n * items: {value: [...], label: [...]}\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n\n let first = [{value: \"\", label: \"(All)\"}];\n let items = util.dataframeToD3(data.items);\n let opts = {\n options: first.concat(items),\n valueField: \"value\",\n labelField: \"label\",\n searchField: \"label\"\n };\n\n let select = $(el).find(\"select\")[0];\n\n let selectize = $(select).selectize(opts)[0].selectize;\n\n let ctHandle = new FilterHandle(data.group);\n\n let lastKnownKeys;\n selectize.on(\"change\", function() {\n if (selectize.items.length === 0) {\n lastKnownKeys = null;\n ctHandle.clear();\n } else {\n let keys = {};\n selectize.items.forEach(function(group) {\n data.map[group].forEach(function(key) {\n keys[key] = true;\n });\n });\n let keyArray = Object.keys(keys);\n keyArray.sort();\n lastKnownKeys = keyArray;\n ctHandle.set(keyArray);\n }\n });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n", + "import * as input from \"./input\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\nlet strftime = global.strftime;\n\ninput.register({\n className: \"crosstalk-input-slider\",\n\n factory: function(el, data) {\n /*\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n let ctHandle = new FilterHandle(data.group);\n\n let opts = {};\n let $el = $(el).find(\"input\");\n let dataType = $el.data(\"data-type\");\n let timeFormat = $el.data(\"time-format\");\n let round = $el.data(\"round\");\n let timeFormatter;\n\n // Set up formatting functions\n if (dataType === \"date\") {\n timeFormatter = strftime.utc();\n opts.prettify = function(num) {\n return timeFormatter(timeFormat, new Date(num));\n };\n\n } else if (dataType === \"datetime\") {\n let timezone = $el.data(\"timezone\");\n if (timezone)\n timeFormatter = strftime.timezone(timezone);\n else\n timeFormatter = strftime;\n\n opts.prettify = function(num) {\n return timeFormatter(timeFormat, new Date(num));\n };\n } else if (dataType === \"number\") {\n if (typeof round !== \"undefined\")\n opts.prettify = function(num) {\n let factor = Math.pow(10, round);\n return Math.round(num * factor) / factor;\n };\n }\n\n $el.ionRangeSlider(opts);\n\n function getValue() {\n let result = $el.data(\"ionRangeSlider\").result;\n\n // Function for converting numeric value from slider to appropriate type.\n let convert;\n let dataType = $el.data(\"data-type\");\n if (dataType === \"date\") {\n convert = function(val) {\n return formatDateUTC(new Date(+val));\n };\n } else if (dataType === \"datetime\") {\n convert = function(val) {\n // Convert ms to s\n return +val / 1000;\n };\n } else {\n convert = function(val) { return +val; };\n }\n\n if ($el.data(\"ionRangeSlider\").options.type === \"double\") {\n return [convert(result.from), convert(result.to)];\n } else {\n return convert(result.from);\n }\n }\n\n let lastKnownKeys = null;\n\n $el.on(\"change.crosstalkSliderInput\", function(event) {\n if (!$el.data(\"updating\") && !$el.data(\"animating\")) {\n let [from, to] = getValue();\n let keys = [];\n for (let i = 0; i < data.values.length; i++) {\n let val = data.values[i];\n if (val >= from && val <= to) {\n keys.push(data.keys[i]);\n }\n }\n keys.sort();\n ctHandle.set(keys);\n lastKnownKeys = keys;\n }\n });\n\n\n // let $el = $(el);\n // $el.on(\"change\", \"input[type=\"checkbox\"]\", function() {\n // let checked = $el.find(\"input[type=\"checkbox\"]:checked\");\n // if (checked.length === 0) {\n // ctHandle.clear();\n // } else {\n // let keys = {};\n // checked.each(function() {\n // data.map[this.value].forEach(function(key) {\n // keys[key] = true;\n // });\n // });\n // let keyArray = Object.keys(keys);\n // keyArray.sort();\n // ctHandle.set(keyArray);\n // }\n // });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n\n\n// Convert a number to a string with leading zeros\nfunction padZeros(n, digits) {\n let str = n.toString();\n while (str.length < digits)\n str = \"0\" + str;\n return str;\n}\n\n// Given a Date object, return a string in yyyy-mm-dd format, using the\n// UTC date. This may be a day off from the date in the local time zone.\nfunction formatDateUTC(date) {\n if (date instanceof Date) {\n return date.getUTCFullYear() + \"-\" +\n padZeros(date.getUTCMonth()+1, 2) + \"-\" +\n padZeros(date.getUTCDate(), 2);\n\n } else {\n return null;\n }\n}\n", + "import Events from \"./events\";\nimport grp from \"./group\";\nimport * as util from \"./util\";\n\n/**\n * Use this class to read and write (and listen for changes to) the selection\n * for a Crosstalk group. This is intended to be used for linked brushing.\n *\n * If two (or more) `SelectionHandle` instances in the same webpage share the\n * same group name, they will share the same state. Setting the selection using\n * one `SelectionHandle` instance will result in the `value` property instantly\n * changing across the others, and `\"change\"` event listeners on all instances\n * (including the one that initiated the sending) will fire.\n *\n * @param {string} [group] - The name of the Crosstalk group, or if none,\n * null or undefined (or any other falsy value). This can be changed later\n * via the [SelectionHandle#setGroup](#setGroup) method.\n * @param {Object} [extraInfo] - An object whose properties will be copied to\n * the event object whenever an event is emitted.\n */\nexport class SelectionHandle {\n\n constructor(group = null, extraInfo = null) {\n this._eventRelay = new Events();\n this._emitter = new util.SubscriptionTracker(this._eventRelay);\n\n // Name of the group we're currently tracking, if any. Can change over time.\n this._group = null;\n // The Var we're currently tracking, if any. Can change over time.\n this._var = null;\n // The event handler subscription we currently have on var.on(\"change\").\n this._varOnChangeSub = null;\n\n this._extraInfo = util.extend({ sender: this }, extraInfo);\n\n this.setGroup(group);\n }\n\n /**\n * Changes the Crosstalk group membership of this SelectionHandle. The group\n * being switched away from (if any) will not have its selection value\n * modified as a result of calling `setGroup`, even if this handle was the\n * most recent handle to set the selection of the group.\n *\n * The group being switched to (if any) will also not have its selection value\n * modified as a result of calling `setGroup`. If you want to set the\n * selection value of the new group, call `set` explicitly.\n *\n * @param {string} group - The name of the Crosstalk group, or null (or\n * undefined) to clear the group.\n */\n setGroup(group) {\n // If group is unchanged, do nothing\n if (this._group === group)\n return;\n // Treat null, undefined, and other falsy values the same\n if (!this._group && !group)\n return;\n\n if (this._var) {\n this._var.off(\"change\", this._varOnChangeSub);\n this._var = null;\n this._varOnChangeSub = null;\n }\n\n this._group = group;\n\n if (group) {\n this._var = grp(group).var(\"selection\");\n let sub = this._var.on(\"change\", (e) => {\n this._eventRelay.trigger(\"change\", e, this);\n });\n this._varOnChangeSub = sub;\n }\n }\n\n /**\n * Retrieves the current selection for the group represented by this\n * `SelectionHandle`.\n *\n * - If no selection is active, then this value will be falsy.\n * - If a selection is active, but no data points are selected, then this\n * value will be an empty array.\n * - If a selection is active, and data points are selected, then the keys\n * of the selected data points will be present in the array.\n */\n get value() {\n return this._var ? this._var.get() : null;\n }\n\n /**\n * Combines the given `extraInfo` (if any) with the handle's default\n * `_extraInfo` (if any).\n * @private\n */\n _mergeExtraInfo(extraInfo) {\n // Important incidental effect: shallow clone is returned\n return util.extend({},\n this._extraInfo ? this._extraInfo : null,\n extraInfo ? extraInfo : null);\n }\n\n /**\n * Overwrites the current selection for the group, and raises the `\"change\"`\n * event among all of the group's '`SelectionHandle` instances (including\n * this one).\n *\n * @fires SelectionHandle#change\n * @param {string[]} selectedKeys - Falsy, empty array, or array of keys (see\n * {@link SelectionHandle#value}).\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `SelectionHandle` constructor).\n */\n set(selectedKeys, extraInfo) {\n if (this._var)\n this._var.set(selectedKeys, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * Overwrites the current selection for the group, and raises the `\"change\"`\n * event among all of the group's '`SelectionHandle` instances (including\n * this one).\n *\n * @fires SelectionHandle#change\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any that were passed\n * into the `SelectionHandle` constructor).\n */\n clear(extraInfo) {\n if (this._var)\n this.set(void 0, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * Subscribes to events on this `SelectionHandle`.\n *\n * @param {string} eventType - Indicates the type of events to listen to.\n * Currently, only `\"change\"` is supported.\n * @param {SelectionHandle~listener} listener - The callback function that\n * will be invoked when the event occurs.\n * @return {string} - A token to pass to {@link SelectionHandle#off} to cancel\n * this subscription.\n */\n on(eventType, listener) {\n return this._emitter.on(eventType, listener);\n }\n\n /**\n * Cancels event subscriptions created by {@link SelectionHandle#on}.\n *\n * @param {string} eventType - The type of event to unsubscribe.\n * @param {string|SelectionHandle~listener} listener - Either the callback\n * function previously passed into {@link SelectionHandle#on}, or the\n * string that was returned from {@link SelectionHandle#on}.\n */\n off(eventType, listener) {\n return this._emitter.off(eventType, listener);\n }\n\n /**\n * Shuts down the `SelectionHandle` object.\n *\n * Removes all event listeners that were added through this handle.\n */\n close() {\n this._emitter.removeAllListeners();\n this.setGroup(null);\n }\n}\n\n/**\n * @callback SelectionHandle~listener\n * @param {Object} event - An object containing details of the event. For\n * `\"change\"` events, this includes the properties `value` (the new\n * value of the selection, or `undefined` if no selection is active),\n * `oldValue` (the previous value of the selection), and `sender` (the\n * `SelectionHandle` instance that made the change).\n */\n\n/**\n * @event SelectionHandle#change\n * @type {object}\n * @property {object} value - The new value of the selection, or `undefined`\n * if no selection is active.\n * @property {object} oldValue - The previous value of the selection.\n * @property {SelectionHandle} sender - The `SelectionHandle` instance that\n * changed the value.\n */\n", + "export function extend(target, ...sources) {\n for (let i = 0; i < sources.length; i++) {\n let src = sources[i];\n if (typeof(src) === \"undefined\" || src === null)\n continue;\n\n for (let key in src) {\n if (src.hasOwnProperty(key)) {\n target[key] = src[key];\n }\n }\n }\n return target;\n}\n\nexport function checkSorted(list) {\n for (let i = 1; i < list.length; i++) {\n if (list[i] <= list[i-1]) {\n throw new Error(\"List is not sorted or contains duplicate\");\n }\n }\n}\n\nexport function diffSortedLists(a, b) {\n let i_a = 0;\n let i_b = 0;\n\n if (!a) a = [];\n if (!b) b = [];\n\n let a_only = [];\n let b_only = [];\n\n checkSorted(a);\n checkSorted(b);\n\n while (i_a < a.length && i_b < b.length) {\n if (a[i_a] === b[i_b]) {\n i_a++;\n i_b++;\n } else if (a[i_a] < b[i_b]) {\n a_only.push(a[i_a++]);\n } else {\n b_only.push(b[i_b++]);\n }\n }\n\n if (i_a < a.length)\n a_only = a_only.concat(a.slice(i_a));\n if (i_b < b.length)\n b_only = b_only.concat(b.slice(i_b));\n return {\n removed: a_only,\n added: b_only\n };\n}\n\n// Convert from wide: { colA: [1,2,3], colB: [4,5,6], ... }\n// to long: [ {colA: 1, colB: 4}, {colA: 2, colB: 5}, ... ]\nexport function dataframeToD3(df) {\n let names = [];\n let length;\n for (let name in df) {\n if (df.hasOwnProperty(name))\n names.push(name);\n if (typeof(df[name]) !== \"object\" || typeof(df[name].length) === \"undefined\") {\n throw new Error(\"All fields must be arrays\");\n } else if (typeof(length) !== \"undefined\" && length !== df[name].length) {\n throw new Error(\"All fields must be arrays of the same length\");\n }\n length = df[name].length;\n }\n let results = [];\n let item;\n for (let row = 0; row < length; row++) {\n item = {};\n for (let col = 0; col < names.length; col++) {\n item[names[col]] = df[names[col]][row];\n }\n results.push(item);\n }\n return results;\n}\n\n/**\n * Keeps track of all event listener additions/removals and lets all active\n * listeners be removed with a single operation.\n *\n * @private\n */\nexport class SubscriptionTracker {\n constructor(emitter) {\n this._emitter = emitter;\n this._subs = {};\n }\n\n on(eventType, listener) {\n let sub = this._emitter.on(eventType, listener);\n this._subs[sub] = eventType;\n return sub;\n }\n\n off(eventType, listener) {\n let sub = this._emitter.off(eventType, listener);\n if (sub) {\n delete this._subs[sub];\n }\n return sub;\n }\n\n removeAllListeners() {\n let current_subs = this._subs;\n this._subs = {};\n Object.keys(current_subs).forEach((sub) => {\n this._emitter.off(current_subs[sub], sub);\n });\n }\n}\n", + "import Events from \"./events\";\n\nexport default class Var {\n constructor(group, name, /*optional*/ value) {\n this._group = group;\n this._name = name;\n this._value = value;\n this._events = new Events();\n }\n\n get() {\n return this._value;\n }\n\n set(value, /*optional*/ event) {\n if (this._value === value) {\n // Do nothing; the value hasn't changed\n return;\n }\n let oldValue = this._value;\n this._value = value;\n // Alert JavaScript listeners that the value has changed\n let evt = {};\n if (event && typeof(event) === \"object\") {\n for (let k in event) {\n if (event.hasOwnProperty(k))\n evt[k] = event[k];\n }\n }\n evt.oldValue = oldValue;\n evt.value = value;\n this._events.trigger(\"change\", evt, this);\n\n // TODO: Make this extensible, to let arbitrary back-ends know that\n // something has changed\n if (global.Shiny && global.Shiny.onInputChange) {\n global.Shiny.onInputChange(\n \".clientValue-\" +\n (this._group.name !== null ? this._group.name + \"-\" : \"\") +\n this._name,\n typeof(value) === \"undefined\" ? null : value\n );\n }\n }\n\n on(eventType, listener) {\n return this._events.on(eventType, listener);\n }\n\n off(eventType, listener) {\n return this._events.off(eventType, listener);\n }\n}\n" + ] +} \ No newline at end of file diff --git a/_freeze/site_libs/crosstalk-1.2.1/js/crosstalk.min.js b/_freeze/site_libs/crosstalk-1.2.1/js/crosstalk.min.js new file mode 100644 index 00000000..b7ec0ac9 --- /dev/null +++ b/_freeze/site_libs/crosstalk-1.2.1/js/crosstalk.min.js @@ -0,0 +1,2 @@ +!function o(u,a,l){function s(n,e){if(!a[n]){if(!u[n]){var t="function"==typeof require&&require;if(!e&&t)return t(n,!0);if(f)return f(n,!0);var r=new Error("Cannot find module '"+n+"'");throw r.code="MODULE_NOT_FOUND",r}var i=a[n]={exports:{}};u[n][0].call(i.exports,function(e){var t=u[n][1][e];return s(t||e)},i,i.exports,o,u,a,l)}return a[n].exports}for(var f="function"==typeof require&&require,e=0;e?@[\\\]^`{|}~])/g,"\\$1")+"']"),r=JSON.parse(n[0].innerText),i=e.factory(t,r);o(t).data("crosstalk-instance",i),o(t).addClass("crosstalk-input-bound")}if(t.Shiny){var e=new t.Shiny.InputBinding,u=t.jQuery;u.extend(e,{find:function(e){return u(e).find(".crosstalk-input")},initialize:function(e){var t,n;u(e).hasClass("crosstalk-input-bound")||(n=o(t=e),Object.keys(r).forEach(function(e){n.hasClass(e)&&!n.hasClass("crosstalk-input-bound")&&i(r[e],t)}))},getId:function(e){return e.id},getValue:function(e){},setValue:function(e,t){},receiveMessage:function(e,t){},subscribe:function(e,t){u(e).data("crosstalk-instance").resume()},unsubscribe:function(e){u(e).data("crosstalk-instance").suspend()}}),t.Shiny.inputBindings.register(e,"crosstalk.inputBinding")}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{}],7:[function(r,e,t){(function(e){"use strict";var t=function(e){{if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}}(r("./input")),n=r("./filter");var a=e.jQuery;t.register({className:"crosstalk-input-checkboxgroup",factory:function(e,r){var i=new n.FilterHandle(r.group),o=void 0,u=a(e);return u.on("change","input[type='checkbox']",function(){var e=u.find("input[type='checkbox']:checked");if(0===e.length)o=null,i.clear();else{var t={};e.each(function(){r.map[this.value].forEach(function(e){t[e]=!0})});var n=Object.keys(t);n.sort(),o=n,i.set(n)}}),{suspend:function(){i.clear()},resume:function(){o&&i.set(o)}}}})}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./filter":2,"./input":6}],8:[function(r,e,t){(function(e){"use strict";var t=n(r("./input")),l=n(r("./util")),s=r("./filter");function n(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}var f=e.jQuery;t.register({className:"crosstalk-input-select",factory:function(e,n){var t=l.dataframeToD3(n.items),r={options:[{value:"",label:"(All)"}].concat(t),valueField:"value",labelField:"label",searchField:"label"},i=f(e).find("select")[0],o=f(i).selectize(r)[0].selectize,u=new s.FilterHandle(n.group),a=void 0;return o.on("change",function(){if(0===o.items.length)a=null,u.clear();else{var t={};o.items.forEach(function(e){n.map[e].forEach(function(e){t[e]=!0})});var e=Object.keys(t);e.sort(),a=e,u.set(e)}}),{suspend:function(){u.clear()},resume:function(){a&&u.set(a)}}}})}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./filter":2,"./input":6,"./util":11}],9:[function(n,e,t){(function(e){"use strict";var d=function(e,t){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return function(e,t){var n=[],r=!0,i=!1,o=void 0;try{for(var u,a=e[Symbol.iterator]();!(r=(u=a.next()).done)&&(n.push(u.value),!t||n.length!==t);r=!0);}catch(e){i=!0,o=e}finally{try{!r&&a.return&&a.return()}finally{if(i)throw o}}return n}(e,t);throw new TypeError("Invalid attempt to destructure non-iterable instance")},t=function(e){{if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}}(n("./input")),a=n("./filter");var v=e.jQuery,p=e.strftime;function y(e,t){for(var n=e.toString();n.length {\n this._eventRelay.trigger(\"change\", e, this);\n });\n this._varOnChangeSub = sub;\n }\n }\n\n /**\n * Combine the given `extraInfo` (if any) with the handle's default\n * `_extraInfo` (if any).\n * @private\n */\n _mergeExtraInfo(extraInfo) {\n return util.extend({},\n this._extraInfo ? this._extraInfo : null,\n extraInfo ? extraInfo : null);\n }\n\n /**\n * Close the handle. This clears this handle's contribution to the filter set,\n * and unsubscribes all event listeners.\n */\n close() {\n this._emitter.removeAllListeners();\n this.clear();\n this.setGroup(null);\n }\n\n /**\n * Clear this handle's contribution to the filter set.\n *\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `FilterHandle` constructor).\n * \n * @fires FilterHandle#change\n */\n clear(extraInfo) {\n if (!this._filterSet)\n return;\n this._filterSet.clear(this._id);\n this._onChange(extraInfo);\n }\n\n /**\n * Set this handle's contribution to the filter set. This array should consist\n * of the keys of the rows that _should_ be displayed; any keys that are not\n * present in the array will be considered _filtered out_. Note that multiple\n * `FilterHandle` instances in the group may each contribute an array of keys,\n * and only those keys that appear in _all_ of the arrays make it through the\n * filter.\n *\n * @param {string[]} keys - Empty array, or array of keys. To clear the\n * filter, don't pass an empty array; instead, use the\n * {@link FilterHandle#clear} method.\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `FilterHandle` constructor).\n * \n * @fires FilterHandle#change\n */\n set(keys, extraInfo) {\n if (!this._filterSet)\n return;\n this._filterSet.update(this._id, keys);\n this._onChange(extraInfo);\n }\n\n /**\n * @return {string[]|null} - Either: 1) an array of keys that made it through\n * all of the `FilterHandle` instances, or, 2) `null`, which means no filter\n * is being applied (all data should be displayed).\n */\n get filteredKeys() {\n return this._filterSet ? this._filterSet.value : null;\n }\n\n /**\n * Subscribe to events on this `FilterHandle`.\n *\n * @param {string} eventType - Indicates the type of events to listen to.\n * Currently, only `\"change\"` is supported.\n * @param {FilterHandle~listener} listener - The callback function that\n * will be invoked when the event occurs.\n * @return {string} - A token to pass to {@link FilterHandle#off} to cancel\n * this subscription.\n */\n on(eventType, listener) {\n return this._emitter.on(eventType, listener);\n }\n\n /**\n * Cancel event subscriptions created by {@link FilterHandle#on}.\n *\n * @param {string} eventType - The type of event to unsubscribe.\n * @param {string|FilterHandle~listener} listener - Either the callback\n * function previously passed into {@link FilterHandle#on}, or the\n * string that was returned from {@link FilterHandle#on}.\n */\n off(eventType, listener) {\n return this._emitter.off(eventType, listener);\n }\n\n _onChange(extraInfo) {\n if (!this._filterSet)\n return;\n this._filterVar.set(this._filterSet.value, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * @callback FilterHandle~listener\n * @param {Object} event - An object containing details of the event. For\n * `\"change\"` events, this includes the properties `value` (the new\n * value of the filter set, or `null` if no filter set is active),\n * `oldValue` (the previous value of the filter set), and `sender` (the\n * `FilterHandle` instance that made the change).\n */\n\n}\n\n/**\n * @event FilterHandle#change\n * @type {object}\n * @property {object} value - The new value of the filter set, or `null`\n * if no filter set is active.\n * @property {object} oldValue - The previous value of the filter set.\n * @property {FilterHandle} sender - The `FilterHandle` instance that\n * changed the value.\n */\n","import { diffSortedLists } from \"./util\";\n\nfunction naturalComparator(a, b) {\n if (a === b) {\n return 0;\n } else if (a < b) {\n return -1;\n } else if (a > b) {\n return 1;\n }\n}\n\n/**\n * @private\n */\nexport default class FilterSet {\n constructor() {\n this.reset();\n }\n\n reset() {\n // Key: handle ID, Value: array of selected keys, or null\n this._handles = {};\n // Key: key string, Value: count of handles that include it\n this._keys = {};\n this._value = null;\n this._activeHandles = 0;\n }\n\n get value() {\n return this._value;\n }\n\n update(handleId, keys) {\n if (keys !== null) {\n keys = keys.slice(0); // clone before sorting\n keys.sort(naturalComparator);\n }\n\n let {added, removed} = diffSortedLists(this._handles[handleId], keys);\n this._handles[handleId] = keys;\n\n for (let i = 0; i < added.length; i++) {\n this._keys[added[i]] = (this._keys[added[i]] || 0) + 1;\n }\n for (let i = 0; i < removed.length; i++) {\n this._keys[removed[i]]--;\n }\n\n this._updateValue(keys);\n }\n\n /**\n * @param {string[]} keys Sorted array of strings that indicate\n * a superset of possible keys.\n * @private\n */\n _updateValue(keys = this._allKeys) {\n let handleCount = Object.keys(this._handles).length;\n if (handleCount === 0) {\n this._value = null;\n } else {\n this._value = [];\n for (let i = 0; i < keys.length; i++) {\n let count = this._keys[keys[i]];\n if (count === handleCount) {\n this._value.push(keys[i]);\n }\n }\n }\n }\n\n clear(handleId) {\n if (typeof(this._handles[handleId]) === \"undefined\") {\n return;\n }\n\n let keys = this._handles[handleId];\n if (!keys) {\n keys = [];\n }\n\n for (let i = 0; i < keys.length; i++) {\n this._keys[keys[i]]--;\n }\n delete this._handles[handleId];\n\n this._updateValue();\n }\n\n get _allKeys() {\n let allKeys = Object.keys(this._keys);\n allKeys.sort(naturalComparator);\n return allKeys;\n }\n}\n","import Var from \"./var\";\n\n// Use a global so that multiple copies of crosstalk.js can be loaded and still\n// have groups behave as singletons across all copies.\nglobal.__crosstalk_groups = global.__crosstalk_groups || {};\nlet groups = global.__crosstalk_groups;\n\nexport default function group(groupName) {\n if (groupName && typeof(groupName) === \"string\") {\n if (!groups.hasOwnProperty(groupName)) {\n groups[groupName] = new Group(groupName);\n }\n return groups[groupName];\n } else if (typeof(groupName) === \"object\" && groupName._vars && groupName.var) {\n // Appears to already be a group object\n return groupName;\n } else if (Array.isArray(groupName) &&\n groupName.length == 1 &&\n typeof(groupName[0]) === \"string\") {\n return group(groupName[0]);\n } else {\n throw new Error(\"Invalid groupName argument\");\n }\n}\n\nclass Group {\n constructor(name) {\n this.name = name;\n this._vars = {};\n }\n\n var(name) {\n if (!name || typeof(name) !== \"string\") {\n throw new Error(\"Invalid var name\");\n }\n\n if (!this._vars.hasOwnProperty(name))\n this._vars[name] = new Var(this, name);\n return this._vars[name];\n }\n\n has(name) {\n if (!name || typeof(name) !== \"string\") {\n throw new Error(\"Invalid var name\");\n }\n\n return this._vars.hasOwnProperty(name);\n }\n}\n","import group from \"./group\";\nimport { SelectionHandle } from \"./selection\";\nimport { FilterHandle } from \"./filter\";\nimport { bind } from \"./input\";\nimport \"./input_selectize\";\nimport \"./input_checkboxgroup\";\nimport \"./input_slider\";\n\nconst defaultGroup = group(\"default\");\n\nfunction var_(name) {\n return defaultGroup.var(name);\n}\n\nfunction has(name) {\n return defaultGroup.has(name);\n}\n\nif (global.Shiny) {\n global.Shiny.addCustomMessageHandler(\"update-client-value\", function(message) {\n if (typeof(message.group) === \"string\") {\n group(message.group).var(message.name).set(message.value);\n } else {\n var_(message.name).set(message.value);\n }\n });\n}\n\nconst crosstalk = {\n group: group,\n var: var_,\n has: has,\n SelectionHandle: SelectionHandle,\n FilterHandle: FilterHandle,\n bind: bind\n};\n\n/**\n * @namespace crosstalk\n */\nexport default crosstalk;\nglobal.crosstalk = crosstalk;\n","let $ = global.jQuery;\n\nlet bindings = {};\n\nexport function register(reg) {\n bindings[reg.className] = reg;\n if (global.document && global.document.readyState !== \"complete\") {\n $(() => {\n bind();\n });\n } else if (global.document) {\n setTimeout(bind, 100);\n }\n}\n\nexport function bind() {\n Object.keys(bindings).forEach(function(className) {\n let binding = bindings[className];\n $(\".\" + binding.className).not(\".crosstalk-input-bound\").each(function(i, el) {\n bindInstance(binding, el);\n });\n });\n}\n\n// Escape jQuery identifier\nfunction $escape(val) {\n return val.replace(/([!\"#$%&'()*+,./:;<=>?@[\\\\\\]^`{|}~])/g, \"\\\\$1\");\n}\n\nfunction bindEl(el) {\n let $el = $(el);\n Object.keys(bindings).forEach(function(className) {\n if ($el.hasClass(className) && !$el.hasClass(\"crosstalk-input-bound\")) {\n let binding = bindings[className];\n bindInstance(binding, el);\n }\n });\n}\n\nfunction bindInstance(binding, el) {\n let jsonEl = $(el).find(\"script[type='application/json'][data-for='\" + $escape(el.id) + \"']\");\n let data = JSON.parse(jsonEl[0].innerText);\n\n let instance = binding.factory(el, data);\n $(el).data(\"crosstalk-instance\", instance);\n $(el).addClass(\"crosstalk-input-bound\");\n}\n\nif (global.Shiny) {\n let inputBinding = new global.Shiny.InputBinding();\n let $ = global.jQuery;\n $.extend(inputBinding, {\n find: function(scope) {\n return $(scope).find(\".crosstalk-input\");\n },\n initialize: function(el) {\n if (!$(el).hasClass(\"crosstalk-input-bound\")) {\n bindEl(el);\n }\n },\n getId: function(el) {\n return el.id;\n },\n getValue: function(el) {\n\n },\n setValue: function(el, value) {\n\n },\n receiveMessage: function(el, data) {\n\n },\n subscribe: function(el, callback) {\n $(el).data(\"crosstalk-instance\").resume();\n },\n unsubscribe: function(el) {\n $(el).data(\"crosstalk-instance\").suspend();\n }\n });\n global.Shiny.inputBindings.register(inputBinding, \"crosstalk.inputBinding\");\n}\n","import * as input from \"./input\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\n\ninput.register({\n className: \"crosstalk-input-checkboxgroup\",\n\n factory: function(el, data) {\n /*\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n let ctHandle = new FilterHandle(data.group);\n\n let lastKnownKeys;\n let $el = $(el);\n $el.on(\"change\", \"input[type='checkbox']\", function() {\n let checked = $el.find(\"input[type='checkbox']:checked\");\n if (checked.length === 0) {\n lastKnownKeys = null;\n ctHandle.clear();\n } else {\n let keys = {};\n checked.each(function() {\n data.map[this.value].forEach(function(key) {\n keys[key] = true;\n });\n });\n let keyArray = Object.keys(keys);\n keyArray.sort();\n lastKnownKeys = keyArray;\n ctHandle.set(keyArray);\n }\n });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n","import * as input from \"./input\";\nimport * as util from \"./util\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\n\ninput.register({\n className: \"crosstalk-input-select\",\n\n factory: function(el, data) {\n /*\n * items: {value: [...], label: [...]}\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n\n let first = [{value: \"\", label: \"(All)\"}];\n let items = util.dataframeToD3(data.items);\n let opts = {\n options: first.concat(items),\n valueField: \"value\",\n labelField: \"label\",\n searchField: \"label\"\n };\n\n let select = $(el).find(\"select\")[0];\n\n let selectize = $(select).selectize(opts)[0].selectize;\n\n let ctHandle = new FilterHandle(data.group);\n\n let lastKnownKeys;\n selectize.on(\"change\", function() {\n if (selectize.items.length === 0) {\n lastKnownKeys = null;\n ctHandle.clear();\n } else {\n let keys = {};\n selectize.items.forEach(function(group) {\n data.map[group].forEach(function(key) {\n keys[key] = true;\n });\n });\n let keyArray = Object.keys(keys);\n keyArray.sort();\n lastKnownKeys = keyArray;\n ctHandle.set(keyArray);\n }\n });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n","import * as input from \"./input\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\nlet strftime = global.strftime;\n\ninput.register({\n className: \"crosstalk-input-slider\",\n\n factory: function(el, data) {\n /*\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n let ctHandle = new FilterHandle(data.group);\n\n let opts = {};\n let $el = $(el).find(\"input\");\n let dataType = $el.data(\"data-type\");\n let timeFormat = $el.data(\"time-format\");\n let round = $el.data(\"round\");\n let timeFormatter;\n\n // Set up formatting functions\n if (dataType === \"date\") {\n timeFormatter = strftime.utc();\n opts.prettify = function(num) {\n return timeFormatter(timeFormat, new Date(num));\n };\n\n } else if (dataType === \"datetime\") {\n let timezone = $el.data(\"timezone\");\n if (timezone)\n timeFormatter = strftime.timezone(timezone);\n else\n timeFormatter = strftime;\n\n opts.prettify = function(num) {\n return timeFormatter(timeFormat, new Date(num));\n };\n } else if (dataType === \"number\") {\n if (typeof round !== \"undefined\")\n opts.prettify = function(num) {\n let factor = Math.pow(10, round);\n return Math.round(num * factor) / factor;\n };\n }\n\n $el.ionRangeSlider(opts);\n\n function getValue() {\n let result = $el.data(\"ionRangeSlider\").result;\n\n // Function for converting numeric value from slider to appropriate type.\n let convert;\n let dataType = $el.data(\"data-type\");\n if (dataType === \"date\") {\n convert = function(val) {\n return formatDateUTC(new Date(+val));\n };\n } else if (dataType === \"datetime\") {\n convert = function(val) {\n // Convert ms to s\n return +val / 1000;\n };\n } else {\n convert = function(val) { return +val; };\n }\n\n if ($el.data(\"ionRangeSlider\").options.type === \"double\") {\n return [convert(result.from), convert(result.to)];\n } else {\n return convert(result.from);\n }\n }\n\n let lastKnownKeys = null;\n\n $el.on(\"change.crosstalkSliderInput\", function(event) {\n if (!$el.data(\"updating\") && !$el.data(\"animating\")) {\n let [from, to] = getValue();\n let keys = [];\n for (let i = 0; i < data.values.length; i++) {\n let val = data.values[i];\n if (val >= from && val <= to) {\n keys.push(data.keys[i]);\n }\n }\n keys.sort();\n ctHandle.set(keys);\n lastKnownKeys = keys;\n }\n });\n\n\n // let $el = $(el);\n // $el.on(\"change\", \"input[type=\"checkbox\"]\", function() {\n // let checked = $el.find(\"input[type=\"checkbox\"]:checked\");\n // if (checked.length === 0) {\n // ctHandle.clear();\n // } else {\n // let keys = {};\n // checked.each(function() {\n // data.map[this.value].forEach(function(key) {\n // keys[key] = true;\n // });\n // });\n // let keyArray = Object.keys(keys);\n // keyArray.sort();\n // ctHandle.set(keyArray);\n // }\n // });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n\n\n// Convert a number to a string with leading zeros\nfunction padZeros(n, digits) {\n let str = n.toString();\n while (str.length < digits)\n str = \"0\" + str;\n return str;\n}\n\n// Given a Date object, return a string in yyyy-mm-dd format, using the\n// UTC date. This may be a day off from the date in the local time zone.\nfunction formatDateUTC(date) {\n if (date instanceof Date) {\n return date.getUTCFullYear() + \"-\" +\n padZeros(date.getUTCMonth()+1, 2) + \"-\" +\n padZeros(date.getUTCDate(), 2);\n\n } else {\n return null;\n }\n}\n","import Events from \"./events\";\nimport grp from \"./group\";\nimport * as util from \"./util\";\n\n/**\n * Use this class to read and write (and listen for changes to) the selection\n * for a Crosstalk group. This is intended to be used for linked brushing.\n *\n * If two (or more) `SelectionHandle` instances in the same webpage share the\n * same group name, they will share the same state. Setting the selection using\n * one `SelectionHandle` instance will result in the `value` property instantly\n * changing across the others, and `\"change\"` event listeners on all instances\n * (including the one that initiated the sending) will fire.\n *\n * @param {string} [group] - The name of the Crosstalk group, or if none,\n * null or undefined (or any other falsy value). This can be changed later\n * via the [SelectionHandle#setGroup](#setGroup) method.\n * @param {Object} [extraInfo] - An object whose properties will be copied to\n * the event object whenever an event is emitted.\n */\nexport class SelectionHandle {\n\n constructor(group = null, extraInfo = null) {\n this._eventRelay = new Events();\n this._emitter = new util.SubscriptionTracker(this._eventRelay);\n\n // Name of the group we're currently tracking, if any. Can change over time.\n this._group = null;\n // The Var we're currently tracking, if any. Can change over time.\n this._var = null;\n // The event handler subscription we currently have on var.on(\"change\").\n this._varOnChangeSub = null;\n\n this._extraInfo = util.extend({ sender: this }, extraInfo);\n\n this.setGroup(group);\n }\n\n /**\n * Changes the Crosstalk group membership of this SelectionHandle. The group\n * being switched away from (if any) will not have its selection value\n * modified as a result of calling `setGroup`, even if this handle was the\n * most recent handle to set the selection of the group.\n *\n * The group being switched to (if any) will also not have its selection value\n * modified as a result of calling `setGroup`. If you want to set the\n * selection value of the new group, call `set` explicitly.\n *\n * @param {string} group - The name of the Crosstalk group, or null (or\n * undefined) to clear the group.\n */\n setGroup(group) {\n // If group is unchanged, do nothing\n if (this._group === group)\n return;\n // Treat null, undefined, and other falsy values the same\n if (!this._group && !group)\n return;\n\n if (this._var) {\n this._var.off(\"change\", this._varOnChangeSub);\n this._var = null;\n this._varOnChangeSub = null;\n }\n\n this._group = group;\n\n if (group) {\n this._var = grp(group).var(\"selection\");\n let sub = this._var.on(\"change\", (e) => {\n this._eventRelay.trigger(\"change\", e, this);\n });\n this._varOnChangeSub = sub;\n }\n }\n\n /**\n * Retrieves the current selection for the group represented by this\n * `SelectionHandle`.\n *\n * - If no selection is active, then this value will be falsy.\n * - If a selection is active, but no data points are selected, then this\n * value will be an empty array.\n * - If a selection is active, and data points are selected, then the keys\n * of the selected data points will be present in the array.\n */\n get value() {\n return this._var ? this._var.get() : null;\n }\n\n /**\n * Combines the given `extraInfo` (if any) with the handle's default\n * `_extraInfo` (if any).\n * @private\n */\n _mergeExtraInfo(extraInfo) {\n // Important incidental effect: shallow clone is returned\n return util.extend({},\n this._extraInfo ? this._extraInfo : null,\n extraInfo ? extraInfo : null);\n }\n\n /**\n * Overwrites the current selection for the group, and raises the `\"change\"`\n * event among all of the group's '`SelectionHandle` instances (including\n * this one).\n *\n * @fires SelectionHandle#change\n * @param {string[]} selectedKeys - Falsy, empty array, or array of keys (see\n * {@link SelectionHandle#value}).\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `SelectionHandle` constructor).\n */\n set(selectedKeys, extraInfo) {\n if (this._var)\n this._var.set(selectedKeys, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * Overwrites the current selection for the group, and raises the `\"change\"`\n * event among all of the group's '`SelectionHandle` instances (including\n * this one).\n *\n * @fires SelectionHandle#change\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any that were passed\n * into the `SelectionHandle` constructor).\n */\n clear(extraInfo) {\n if (this._var)\n this.set(void 0, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * Subscribes to events on this `SelectionHandle`.\n *\n * @param {string} eventType - Indicates the type of events to listen to.\n * Currently, only `\"change\"` is supported.\n * @param {SelectionHandle~listener} listener - The callback function that\n * will be invoked when the event occurs.\n * @return {string} - A token to pass to {@link SelectionHandle#off} to cancel\n * this subscription.\n */\n on(eventType, listener) {\n return this._emitter.on(eventType, listener);\n }\n\n /**\n * Cancels event subscriptions created by {@link SelectionHandle#on}.\n *\n * @param {string} eventType - The type of event to unsubscribe.\n * @param {string|SelectionHandle~listener} listener - Either the callback\n * function previously passed into {@link SelectionHandle#on}, or the\n * string that was returned from {@link SelectionHandle#on}.\n */\n off(eventType, listener) {\n return this._emitter.off(eventType, listener);\n }\n\n /**\n * Shuts down the `SelectionHandle` object.\n *\n * Removes all event listeners that were added through this handle.\n */\n close() {\n this._emitter.removeAllListeners();\n this.setGroup(null);\n }\n}\n\n/**\n * @callback SelectionHandle~listener\n * @param {Object} event - An object containing details of the event. For\n * `\"change\"` events, this includes the properties `value` (the new\n * value of the selection, or `undefined` if no selection is active),\n * `oldValue` (the previous value of the selection), and `sender` (the\n * `SelectionHandle` instance that made the change).\n */\n\n/**\n * @event SelectionHandle#change\n * @type {object}\n * @property {object} value - The new value of the selection, or `undefined`\n * if no selection is active.\n * @property {object} oldValue - The previous value of the selection.\n * @property {SelectionHandle} sender - The `SelectionHandle` instance that\n * changed the value.\n */\n","export function extend(target, ...sources) {\n for (let i = 0; i < sources.length; i++) {\n let src = sources[i];\n if (typeof(src) === \"undefined\" || src === null)\n continue;\n\n for (let key in src) {\n if (src.hasOwnProperty(key)) {\n target[key] = src[key];\n }\n }\n }\n return target;\n}\n\nexport function checkSorted(list) {\n for (let i = 1; i < list.length; i++) {\n if (list[i] <= list[i-1]) {\n throw new Error(\"List is not sorted or contains duplicate\");\n }\n }\n}\n\nexport function diffSortedLists(a, b) {\n let i_a = 0;\n let i_b = 0;\n\n if (!a) a = [];\n if (!b) b = [];\n\n let a_only = [];\n let b_only = [];\n\n checkSorted(a);\n checkSorted(b);\n\n while (i_a < a.length && i_b < b.length) {\n if (a[i_a] === b[i_b]) {\n i_a++;\n i_b++;\n } else if (a[i_a] < b[i_b]) {\n a_only.push(a[i_a++]);\n } else {\n b_only.push(b[i_b++]);\n }\n }\n\n if (i_a < a.length)\n a_only = a_only.concat(a.slice(i_a));\n if (i_b < b.length)\n b_only = b_only.concat(b.slice(i_b));\n return {\n removed: a_only,\n added: b_only\n };\n}\n\n// Convert from wide: { colA: [1,2,3], colB: [4,5,6], ... }\n// to long: [ {colA: 1, colB: 4}, {colA: 2, colB: 5}, ... ]\nexport function dataframeToD3(df) {\n let names = [];\n let length;\n for (let name in df) {\n if (df.hasOwnProperty(name))\n names.push(name);\n if (typeof(df[name]) !== \"object\" || typeof(df[name].length) === \"undefined\") {\n throw new Error(\"All fields must be arrays\");\n } else if (typeof(length) !== \"undefined\" && length !== df[name].length) {\n throw new Error(\"All fields must be arrays of the same length\");\n }\n length = df[name].length;\n }\n let results = [];\n let item;\n for (let row = 0; row < length; row++) {\n item = {};\n for (let col = 0; col < names.length; col++) {\n item[names[col]] = df[names[col]][row];\n }\n results.push(item);\n }\n return results;\n}\n\n/**\n * Keeps track of all event listener additions/removals and lets all active\n * listeners be removed with a single operation.\n *\n * @private\n */\nexport class SubscriptionTracker {\n constructor(emitter) {\n this._emitter = emitter;\n this._subs = {};\n }\n\n on(eventType, listener) {\n let sub = this._emitter.on(eventType, listener);\n this._subs[sub] = eventType;\n return sub;\n }\n\n off(eventType, listener) {\n let sub = this._emitter.off(eventType, listener);\n if (sub) {\n delete this._subs[sub];\n }\n return sub;\n }\n\n removeAllListeners() {\n let current_subs = this._subs;\n this._subs = {};\n Object.keys(current_subs).forEach((sub) => {\n this._emitter.off(current_subs[sub], sub);\n });\n }\n}\n","import Events from \"./events\";\n\nexport default class Var {\n constructor(group, name, /*optional*/ value) {\n this._group = group;\n this._name = name;\n this._value = value;\n this._events = new Events();\n }\n\n get() {\n return this._value;\n }\n\n set(value, /*optional*/ event) {\n if (this._value === value) {\n // Do nothing; the value hasn't changed\n return;\n }\n let oldValue = this._value;\n this._value = value;\n // Alert JavaScript listeners that the value has changed\n let evt = {};\n if (event && typeof(event) === \"object\") {\n for (let k in event) {\n if (event.hasOwnProperty(k))\n evt[k] = event[k];\n }\n }\n evt.oldValue = oldValue;\n evt.value = value;\n this._events.trigger(\"change\", evt, this);\n\n // TODO: Make this extensible, to let arbitrary back-ends know that\n // something has changed\n if (global.Shiny && global.Shiny.onInputChange) {\n global.Shiny.onInputChange(\n \".clientValue-\" +\n (this._group.name !== null ? this._group.name + \"-\" : \"\") +\n this._name,\n typeof(value) === \"undefined\" ? null : value\n );\n }\n }\n\n on(eventType, listener) {\n return this._events.on(eventType, listener);\n }\n\n off(eventType, listener) {\n return this._events.off(eventType, listener);\n }\n}\n"]} \ No newline at end of file diff --git a/_freeze/site_libs/crosstalk-1.2.1/scss/crosstalk.scss b/_freeze/site_libs/crosstalk-1.2.1/scss/crosstalk.scss new file mode 100644 index 00000000..35665616 --- /dev/null +++ b/_freeze/site_libs/crosstalk-1.2.1/scss/crosstalk.scss @@ -0,0 +1,75 @@ +/* Adjust margins outwards, so column contents line up with the edges of the + parent of container-fluid. */ +.container-fluid.crosstalk-bscols { + margin-left: -30px; + margin-right: -30px; + white-space: normal; +} + +/* But don't adjust the margins outwards if we're directly under the body, + i.e. we were the top-level of something at the console. */ +body > .container-fluid.crosstalk-bscols { + margin-left: auto; + margin-right: auto; +} + +.crosstalk-input-checkboxgroup .crosstalk-options-group .crosstalk-options-column { + display: inline-block; + padding-right: 12px; + vertical-align: top; +} + +@media only screen and (max-width:480px) { + .crosstalk-input-checkboxgroup .crosstalk-options-group .crosstalk-options-column { + display: block; + padding-right: inherit; + } +} + +/* Relevant BS3 styles to make filter_checkbox() look reasonable without Bootstrap */ +.crosstalk-input { + margin-bottom: 15px; /* a la .form-group */ + .control-label { + margin-bottom: 0; + vertical-align: middle; + } + input[type="checkbox"] { + margin: 4px 0 0; + margin-top: 1px; + line-height: normal; + } + .checkbox { + position: relative; + display: block; + margin-top: 10px; + margin-bottom: 10px; + } + .checkbox > label{ + padding-left: 20px; + margin-bottom: 0; + font-weight: 400; + cursor: pointer; + } + .checkbox input[type="checkbox"], + .checkbox-inline input[type="checkbox"] { + position: absolute; + margin-top: 2px; + margin-left: -20px; + } + .checkbox + .checkbox { + margin-top: -5px; + } + .checkbox-inline { + position: relative; + display: inline-block; + padding-left: 20px; + margin-bottom: 0; + font-weight: 400; + vertical-align: middle; + cursor: pointer; + } + .checkbox-inline + .checkbox-inline { + margin-top: 0; + margin-left: 10px; + } +} diff --git a/_freeze/site_libs/datatables-binding-0.33/datatables.js b/_freeze/site_libs/datatables-binding-0.33/datatables.js new file mode 100644 index 00000000..765b53cb --- /dev/null +++ b/_freeze/site_libs/datatables-binding-0.33/datatables.js @@ -0,0 +1,1539 @@ +(function() { + +// some helper functions: using a global object DTWidget so that it can be used +// in JS() code, e.g. datatable(options = list(foo = JS('code'))); unlike R's +// dynamic scoping, when 'code' is eval'ed, JavaScript does not know objects +// from the "parent frame", e.g. JS('DTWidget') will not work unless it was made +// a global object +var DTWidget = {}; + +// 123456666.7890 -> 123,456,666.7890 +var markInterval = function(d, digits, interval, mark, decMark, precision) { + x = precision ? d.toPrecision(digits) : d.toFixed(digits); + if (!/^-?[\d.]+$/.test(x)) return x; + var xv = x.split('.'); + if (xv.length > 2) return x; // should have at most one decimal point + xv[0] = xv[0].replace(new RegExp('\\B(?=(\\d{' + interval + '})+(?!\\d))', 'g'), mark); + return xv.join(decMark); +}; + +DTWidget.formatCurrency = function(data, currency, digits, interval, mark, decMark, before, zeroPrint) { + var d = parseFloat(data); + if (isNaN(d)) return ''; + if (zeroPrint !== null && d === 0.0) return zeroPrint; + var res = markInterval(d, digits, interval, mark, decMark); + res = before ? (/^-/.test(res) ? '-' + currency + res.replace(/^-/, '') : currency + res) : + res + currency; + return res; +}; + +DTWidget.formatString = function(data, prefix, suffix) { + var d = data; + if (d === null) return ''; + return prefix + d + suffix; +}; + +DTWidget.formatPercentage = function(data, digits, interval, mark, decMark, zeroPrint) { + var d = parseFloat(data); + if (isNaN(d)) return ''; + if (zeroPrint !== null && d === 0.0) return zeroPrint; + return markInterval(d * 100, digits, interval, mark, decMark) + '%'; +}; + +DTWidget.formatRound = function(data, digits, interval, mark, decMark, zeroPrint) { + var d = parseFloat(data); + if (isNaN(d)) return ''; + if (zeroPrint !== null && d === 0.0) return zeroPrint; + return markInterval(d, digits, interval, mark, decMark); +}; + +DTWidget.formatSignif = function(data, digits, interval, mark, decMark, zeroPrint) { + var d = parseFloat(data); + if (isNaN(d)) return ''; + if (zeroPrint !== null && d === 0.0) return zeroPrint; + return markInterval(d, digits, interval, mark, decMark, true); +}; + +DTWidget.formatDate = function(data, method, params) { + var d = data; + if (d === null) return ''; + // (new Date('2015-10-28')).toDateString() may return 2015-10-27 because the + // actual time created could be like 'Tue Oct 27 2015 19:00:00 GMT-0500 (CDT)', + // i.e. the date-only string is treated as UTC time instead of local time + if ((method === 'toDateString' || method === 'toLocaleDateString') && /^\d{4,}\D\d{2}\D\d{2}$/.test(d)) { + d = d.split(/\D/); + d = new Date(d[0], d[1] - 1, d[2]); + } else { + d = new Date(d); + } + return d[method].apply(d, params); +}; + +window.DTWidget = DTWidget; + +// A helper function to update the properties of existing filters +var setFilterProps = function(td, props) { + // Update enabled/disabled state + var $input = $(td).find('input').first(); + var searchable = $input.data('searchable'); + $input.prop('disabled', !searchable || props.disabled); + + // Based on the filter type, set its new values + var type = td.getAttribute('data-type'); + if (['factor', 'logical'].includes(type)) { + // Reformat the new dropdown options for use with selectize + var new_vals = props.params.options.map(function(item) { + return { text: item, value: item }; + }); + + // Find the selectize object + var dropdown = $(td).find('.selectized').eq(0)[0].selectize; + + // Note the current values + var old_vals = dropdown.getValue(); + + // Remove the existing values + dropdown.clearOptions(); + + // Add the new options + dropdown.addOption(new_vals); + + // Preserve the existing values + dropdown.setValue(old_vals); + + } else if (['number', 'integer', 'date', 'time'].includes(type)) { + // Apply internal scaling to new limits. Updating scale not yet implemented. + var slider = $(td).find('.noUi-target').eq(0); + var scale = Math.pow(10, Math.max(0, +slider.data('scale') || 0)); + var new_vals = [props.params.min * scale, props.params.max * scale]; + + // Note what the new limits will be just for this filter + var new_lims = new_vals.slice(); + + // Determine the current values and limits + var old_vals = slider.val().map(Number); + var old_lims = slider.noUiSlider('options').range; + old_lims = [old_lims.min, old_lims.max]; + + // Preserve the current values if filters have been applied; otherwise, apply no filtering + if (old_vals[0] != old_lims[0]) { + new_vals[0] = Math.max(old_vals[0], new_vals[0]); + } + + if (old_vals[1] != old_lims[1]) { + new_vals[1] = Math.min(old_vals[1], new_vals[1]); + } + + // Update the endpoints of the slider + slider.noUiSlider({ + start: new_vals, + range: {'min': new_lims[0], 'max': new_lims[1]} + }, true); + } +}; + +var transposeArray2D = function(a) { + return a.length === 0 ? a : HTMLWidgets.transposeArray2D(a); +}; + +var crosstalkPluginsInstalled = false; + +function maybeInstallCrosstalkPlugins() { + if (crosstalkPluginsInstalled) + return; + crosstalkPluginsInstalled = true; + + $.fn.dataTable.ext.afnFiltering.push( + function(oSettings, aData, iDataIndex) { + var ctfilter = oSettings.nTable.ctfilter; + if (ctfilter && !ctfilter[iDataIndex]) + return false; + + var ctselect = oSettings.nTable.ctselect; + if (ctselect && !ctselect[iDataIndex]) + return false; + + return true; + } + ); +} + +HTMLWidgets.widget({ + name: "datatables", + type: "output", + renderOnNullValue: true, + initialize: function(el, width, height) { + // in order that the type=number inputs return a number + $.valHooks.number = { + get: function(el) { + var value = parseFloat(el.value); + return isNaN(value) ? "" : value; + } + }; + $(el).html(' '); + return { + data: null, + ctfilterHandle: new crosstalk.FilterHandle(), + ctfilterSubscription: null, + ctselectHandle: new crosstalk.SelectionHandle(), + ctselectSubscription: null + }; + }, + renderValue: function(el, data, instance) { + if (el.offsetWidth === 0 || el.offsetHeight === 0) { + instance.data = data; + return; + } + instance.data = null; + var $el = $(el); + $el.empty(); + + if (data === null) { + $el.append(' '); + // clear previous Shiny inputs (if any) + for (var i in instance.clearInputs) instance.clearInputs[i](); + instance.clearInputs = {}; + return; + } + + var crosstalkOptions = data.crosstalkOptions; + if (!crosstalkOptions) crosstalkOptions = { + 'key': null, 'group': null + }; + if (crosstalkOptions.group) { + maybeInstallCrosstalkPlugins(); + instance.ctfilterHandle.setGroup(crosstalkOptions.group); + instance.ctselectHandle.setGroup(crosstalkOptions.group); + } + + // if we are in the viewer then we always want to fillContainer and + // and autoHideNavigation (unless the user has explicitly set these) + if (window.HTMLWidgets.viewerMode) { + if (!data.hasOwnProperty("fillContainer")) + data.fillContainer = true; + if (!data.hasOwnProperty("autoHideNavigation")) + data.autoHideNavigation = true; + } + + // propagate fillContainer to instance (so we have it in resize) + instance.fillContainer = data.fillContainer; + + var cells = data.data; + + if (cells instanceof Array) cells = transposeArray2D(cells); + + $el.append(data.container); + var $table = $el.find('table'); + if (data.class) $table.addClass(data.class); + if (data.caption) $table.prepend(data.caption); + + if (!data.selection) data.selection = { + mode: 'none', selected: null, target: 'row', selectable: null + }; + if (HTMLWidgets.shinyMode && data.selection.mode !== 'none' && + data.selection.target === 'row+column') { + if ($table.children('tfoot').length === 0) { + $table.append($('')); + $table.find('thead tr').clone().appendTo($table.find('tfoot')); + } + } + + // column filters + var filterRow; + switch (data.filter) { + case 'top': + $table.children('thead').append(data.filterHTML); + filterRow = $table.find('thead tr:last td'); + break; + case 'bottom': + if ($table.children('tfoot').length === 0) { + $table.append($('')); + } + $table.children('tfoot').prepend(data.filterHTML); + filterRow = $table.find('tfoot tr:first td'); + break; + } + + var options = { searchDelay: 1000 }; + if (cells !== null) $.extend(options, { + data: cells + }); + + // options for fillContainer + var bootstrapActive = typeof($.fn.popover) != 'undefined'; + if (instance.fillContainer) { + + // force scrollX/scrollY and turn off autoWidth + options.scrollX = true; + options.scrollY = "100px"; // can be any value, we'll adjust below + + // if we aren't paginating then move around the info/filter controls + // to save space at the bottom and rephrase the info callback + if (data.options.paging === false) { + + // we know how to do this cleanly for bootstrap, not so much + // for other themes/layouts + if (bootstrapActive) { + options.dom = "<'row'<'col-sm-4'i><'col-sm-8'f>>" + + "<'row'<'col-sm-12'tr>>"; + } + + options.fnInfoCallback = function(oSettings, iStart, iEnd, + iMax, iTotal, sPre) { + return Number(iTotal).toLocaleString() + " records"; + }; + } + } + + // auto hide navigation if requested + // Note, this only works on client-side processing mode as on server-side, + // cells (data.data) is null; In addition, we require the pageLength option + // being provided explicitly to enable this. Despite we may be able to deduce + // the default value of pageLength, it may complicate things so we'd rather + // put this responsiblity to users and warn them on the R side. + if (data.autoHideNavigation === true && data.options.paging !== false) { + // strip all nav if length >= cells + if ((cells instanceof Array) && data.options.pageLength >= cells.length) + options.dom = bootstrapActive ? "<'row'<'col-sm-12'tr>>" : "t"; + // alternatively lean things out for flexdashboard mobile portrait + else if (bootstrapActive && window.FlexDashboard && window.FlexDashboard.isMobilePhone()) + options.dom = "<'row'<'col-sm-12'f>>" + + "<'row'<'col-sm-12'tr>>" + + "<'row'<'col-sm-12'p>>"; + } + + $.extend(true, options, data.options || {}); + + var searchCols = options.searchCols; + if (searchCols) { + searchCols = searchCols.map(function(x) { + return x === null ? '' : x.search; + }); + // FIXME: this means I don't respect the escapeRegex setting + delete options.searchCols; + } + + // server-side processing? + var server = options.serverSide === true; + + // use the dataSrc function to pre-process JSON data returned from R + var DT_rows_all = [], DT_rows_current = []; + if (server && HTMLWidgets.shinyMode && typeof options.ajax === 'object' && + /^session\/[\da-z]+\/dataobj/.test(options.ajax.url) && !options.ajax.dataSrc) { + options.ajax.dataSrc = function(json) { + DT_rows_all = $.makeArray(json.DT_rows_all); + DT_rows_current = $.makeArray(json.DT_rows_current); + var data = json.data; + if (!colReorderEnabled()) return data; + var table = $table.DataTable(), order = table.colReorder.order(), flag = true, i, j, row; + for (i = 0; i < order.length; ++i) if (order[i] !== i) flag = false; + if (flag) return data; + for (i = 0; i < data.length; ++i) { + row = data[i].slice(); + for (j = 0; j < order.length; ++j) data[i][j] = row[order[j]]; + } + return data; + }; + } + + var thiz = this; + if (instance.fillContainer) $table.on('init.dt', function(e) { + thiz.fillAvailableHeight(el, $(el).innerHeight()); + }); + // If the page contains serveral datatables and one of which enables colReorder, + // the table.colReorder.order() function will exist but throws error when called. + // So it seems like the only way to know if colReorder is enabled or not is to + // check the options. + var colReorderEnabled = function() { return "colReorder" in options; }; + var table = $table.DataTable(options); + $el.data('datatable', table); + + if ('rowGroup' in options) { + // Maintain RowGroup dataSrc when columns are reordered (#1109) + table.on('column-reorder', function(e, settings, details) { + var oldDataSrc = table.rowGroup().dataSrc(); + var newDataSrc = details.mapping[oldDataSrc]; + table.rowGroup().dataSrc(newDataSrc); + }); + } + + // Unregister previous Crosstalk event subscriptions, if they exist + if (instance.ctfilterSubscription) { + instance.ctfilterHandle.off("change", instance.ctfilterSubscription); + instance.ctfilterSubscription = null; + } + if (instance.ctselectSubscription) { + instance.ctselectHandle.off("change", instance.ctselectSubscription); + instance.ctselectSubscription = null; + } + + if (!crosstalkOptions.group) { + $table[0].ctfilter = null; + $table[0].ctselect = null; + } else { + var key = crosstalkOptions.key; + function keysToMatches(keys) { + if (!keys) { + return null; + } else { + var selectedKeys = {}; + for (var i = 0; i < keys.length; i++) { + selectedKeys[keys[i]] = true; + } + var matches = {}; + for (var j = 0; j < key.length; j++) { + if (selectedKeys[key[j]]) + matches[j] = true; + } + return matches; + } + } + + function applyCrosstalkFilter(e) { + $table[0].ctfilter = keysToMatches(e.value); + table.draw(); + } + instance.ctfilterSubscription = instance.ctfilterHandle.on("change", applyCrosstalkFilter); + applyCrosstalkFilter({value: instance.ctfilterHandle.filteredKeys}); + + function applyCrosstalkSelection(e) { + if (e.sender !== instance.ctselectHandle) { + table + .rows('.' + selClass, {search: 'applied'}) + .nodes() + .to$() + .removeClass(selClass); + if (selectedRows) + changeInput('rows_selected', selectedRows(), void 0, true); + } + + if (e.sender !== instance.ctselectHandle && e.value && e.value.length) { + var matches = keysToMatches(e.value); + + // persistent selection with plotly (& leaflet) + var ctOpts = crosstalk.var("plotlyCrosstalkOpts").get() || {}; + if (ctOpts.persistent === true) { + var matches = $.extend(matches, $table[0].ctselect); + } + + $table[0].ctselect = matches; + table.draw(); + } else { + if ($table[0].ctselect) { + $table[0].ctselect = null; + table.draw(); + } + } + } + instance.ctselectSubscription = instance.ctselectHandle.on("change", applyCrosstalkSelection); + // TODO: This next line doesn't seem to work when renderDataTable is used + applyCrosstalkSelection({value: instance.ctselectHandle.value}); + } + + var inArray = function(val, array) { + return $.inArray(val, $.makeArray(array)) > -1; + }; + + // search the i-th column + var searchColumn = function(i, value) { + var regex = false, ci = true; + if (options.search) { + regex = options.search.regex, + ci = options.search.caseInsensitive !== false; + } + // need to transpose the column index when colReorder is enabled + if (table.colReorder) i = table.colReorder.transpose(i); + return table.column(i).search(value, regex, !regex, ci); + }; + + if (data.filter !== 'none') { + if (!data.hasOwnProperty('filterSettings')) data.filterSettings = {}; + + filterRow.each(function(i, td) { + + var $td = $(td), type = $td.data('type'), filter; + var $input = $td.children('div').first().children('input'); + var disabled = $input.prop('disabled'); + var searchable = table.settings()[0].aoColumns[i].bSearchable; + $input.prop('disabled', !searchable || disabled); + $input.data('searchable', searchable); // for updating later + $input.on('input blur', function() { + $input.next('span').toggle(Boolean($input.val())); + }); + // Bootstrap sets pointer-events to none and we won't be able to click + // the clear button + $input.next('span').css('pointer-events', 'auto').hide().click(function() { + $(this).hide().prev('input').val('').trigger('input').focus(); + }); + var searchCol; // search string for this column + if (searchCols && searchCols[i]) { + searchCol = searchCols[i]; + $input.val(searchCol).trigger('input'); + } + var $x = $td.children('div').last(); + + // remove the overflow: hidden attribute of the scrollHead + // (otherwise the scrolling table body obscures the filters) + // The workaround and the discussion from + // https://github.com/rstudio/DT/issues/554#issuecomment-518007347 + // Otherwise the filter selection will not be anchored to the values + // when the columns number is many and scrollX is enabled. + var scrollHead = $(el).find('.dataTables_scrollHead,.dataTables_scrollFoot'); + var cssOverflowHead = scrollHead.css('overflow'); + var scrollBody = $(el).find('.dataTables_scrollBody'); + var cssOverflowBody = scrollBody.css('overflow'); + var scrollTable = $(el).find('.dataTables_scroll'); + var cssOverflowTable = scrollTable.css('overflow'); + if (cssOverflowHead === 'hidden') { + $x.on('show hide', function(e) { + if (e.type === 'show') { + scrollHead.css('overflow', 'visible'); + scrollBody.css('overflow', 'visible'); + scrollTable.css('overflow-x', 'scroll'); + } else { + scrollHead.css('overflow', cssOverflowHead); + scrollBody.css('overflow', cssOverflowBody); + scrollTable.css('overflow-x', cssOverflowTable); + } + }); + $x.css('z-index', 25); + } + + if (inArray(type, ['factor', 'logical'])) { + $input.on({ + click: function() { + $input.parent().hide(); $x.show().trigger('show'); filter[0].selectize.focus(); + }, + input: function() { + var v1 = JSON.stringify(filter[0].selectize.getValue()), v2 = $input.val(); + if (v1 === '[]') v1 = ''; + if (v1 !== v2) filter[0].selectize.setValue(v2 === '' ? [] : JSON.parse(v2)); + } + }); + var $input2 = $x.children('select'); + filter = $input2.selectize($.extend({ + options: $input2.data('options').map(function(v, i) { + return ({text: v, value: v}); + }), + plugins: ['remove_button'], + hideSelected: true, + onChange: function(value) { + if (value === null) value = []; // compatibility with jQuery 3.0 + $input.val(value.length ? JSON.stringify(value) : ''); + if (value.length) $input.trigger('input'); + $input.attr('title', $input.val()); + if (server) { + searchColumn(i, value.length ? JSON.stringify(value) : '').draw(); + return; + } + // turn off filter if nothing selected + $td.data('filter', value.length > 0); + table.draw(); // redraw table, and filters will be applied + } + }, data.filterSettings.select)); + filter[0].selectize.on('blur', function() { + $x.hide().trigger('hide'); $input.parent().show(); $input.trigger('blur'); + }); + filter.next('div').css('margin-bottom', 'auto'); + } else if (type === 'character') { + var fun = function() { + searchColumn(i, $input.val()).draw(); + }; + // throttle searching for server-side processing + var throttledFun = $.fn.dataTable.util.throttle(fun, options.searchDelay); + $input.on('input', function(e, immediate) { + // always bypass throttling when immediate = true (via the updateSearch method) + (immediate || !server) ? fun() : throttledFun(); + }); + } else if (inArray(type, ['number', 'integer', 'date', 'time'])) { + var $x0 = $x; + $x = $x0.children('div').first(); + $x0.css({ + 'background-color': '#fff', + 'border': '1px #ddd solid', + 'border-radius': '4px', + 'padding': data.vertical ? '35px 20px': '20px 20px 10px 20px' + }); + var $spans = $x0.children('span').css({ + 'margin-top': data.vertical ? '0' : '10px', + 'white-space': 'nowrap' + }); + var $span1 = $spans.first(), $span2 = $spans.last(); + var r1 = +$x.data('min'), r2 = +$x.data('max'); + // when the numbers are too small or have many decimal places, the + // slider may have numeric precision problems (#150) + var scale = Math.pow(10, Math.max(0, +$x.data('scale') || 0)); + r1 = Math.round(r1 * scale); r2 = Math.round(r2 * scale); + var scaleBack = function(x, scale) { + if (scale === 1) return x; + var d = Math.round(Math.log(scale) / Math.log(10)); + // to avoid problems like 3.423/100 -> 0.034230000000000003 + return (x / scale).toFixed(d); + }; + var slider_min = function() { + return filter.noUiSlider('options').range.min; + }; + var slider_max = function() { + return filter.noUiSlider('options').range.max; + }; + $input.on({ + focus: function() { + $x0.show().trigger('show'); + // first, make sure the slider div leaves at least 20px between + // the two (slider value) span's + $x0.width(Math.max(160, $span1.outerWidth() + $span2.outerWidth() + 20)); + // then, if the input is really wide or slider is vertical, + // make the slider the same width as the input + if ($x0.outerWidth() < $input.outerWidth() || data.vertical) { + $x0.outerWidth($input.outerWidth()); + } + // make sure the slider div does not reach beyond the right margin + if ($(window).width() < $x0.offset().left + $x0.width()) { + $x0.offset({ + 'left': $input.offset().left + $input.outerWidth() - $x0.outerWidth() + }); + } + }, + blur: function() { + $x0.hide().trigger('hide'); + }, + input: function() { + if ($input.val() === '') filter.val([slider_min(), slider_max()]); + }, + change: function() { + var v = $input.val().replace(/\s/g, ''); + if (v === '') return; + v = v.split('...'); + if (v.length !== 2) { + $input.parent().addClass('has-error'); + return; + } + if (v[0] === '') v[0] = slider_min(); + if (v[1] === '') v[1] = slider_max(); + $input.parent().removeClass('has-error'); + // treat date as UTC time at midnight + var strTime = function(x) { + var s = type === 'date' ? 'T00:00:00Z' : ''; + var t = new Date(x + s).getTime(); + // add 10 minutes to date since it does not hurt the date, and + // it helps avoid the tricky floating point arithmetic problems, + // e.g. sometimes the date may be a few milliseconds earlier + // than the midnight due to precision problems in noUiSlider + return type === 'date' ? t + 3600000 : t; + }; + if (inArray(type, ['date', 'time'])) { + v[0] = strTime(v[0]); + v[1] = strTime(v[1]); + } + if (v[0] != slider_min()) v[0] *= scale; + if (v[1] != slider_max()) v[1] *= scale; + filter.val(v); + } + }); + var formatDate = function(d) { + d = scaleBack(d, scale); + if (type === 'number') return d; + if (type === 'integer') return parseInt(d); + var x = new Date(+d); + if (type === 'date') { + var pad0 = function(x) { + return ('0' + x).substr(-2, 2); + }; + return x.getUTCFullYear() + '-' + pad0(1 + x.getUTCMonth()) + + '-' + pad0(x.getUTCDate()); + } else { + return x.toISOString(); + } + }; + var opts = type === 'date' ? { step: 60 * 60 * 1000 } : + type === 'integer' ? { step: 1 } : {}; + + opts.orientation = data.vertical ? 'vertical': 'horizontal'; + opts.direction = data.vertical ? 'rtl': 'ltr'; + + filter = $x.noUiSlider($.extend({ + start: [r1, r2], + range: {min: r1, max: r2}, + connect: true + }, opts, data.filterSettings.slider)); + if (scale > 1) (function() { + var t1 = r1, t2 = r2; + var val = filter.val(); + while (val[0] > r1 || val[1] < r2) { + if (val[0] > r1) { + t1 -= val[0] - r1; + } + if (val[1] < r2) { + t2 += r2 - val[1]; + } + filter = $x.noUiSlider($.extend({ + start: [t1, t2], + range: {min: t1, max: t2}, + connect: true + }, opts, data.filterSettings.slider), true); + val = filter.val(); + } + r1 = t1; r2 = t2; + })(); + // format with active column renderer, if defined + var colDef = data.options.columnDefs.find(function(def) { + return (def.targets === i || inArray(i, def.targets)) && 'render' in def; + }); + var updateSliderText = function(v1, v2) { + // we only know how to use function renderers + if (colDef && typeof colDef.render === 'function') { + var restore = function(v) { + v = scaleBack(v, scale); + return inArray(type, ['date', 'time']) ? new Date(+v) : v; + } + $span1.text(colDef.render(restore(v1), 'display')); + $span2.text(colDef.render(restore(v2), 'display')); + } else { + $span1.text(formatDate(v1)); + $span2.text(formatDate(v2)); + } + }; + updateSliderText(r1, r2); + var updateSlider = function(e) { + var val = filter.val(); + // turn off filter if in full range + $td.data('filter', val[0] > slider_min() || val[1] < slider_max()); + var v1 = formatDate(val[0]), v2 = formatDate(val[1]), ival; + if ($td.data('filter')) { + ival = v1 + ' ... ' + v2; + $input.attr('title', ival).val(ival).trigger('input'); + } else { + $input.attr('title', '').val(''); + } + updateSliderText(val[0], val[1]); + if (e.type === 'slide') return; // no searching when sliding only + if (server) { + searchColumn(i, $td.data('filter') ? ival : '').draw(); + return; + } + table.draw(); + }; + filter.on({ + set: updateSlider, + slide: updateSlider + }); + } + + // server-side processing will be handled by R (or whatever server + // language you use); the following code is only needed for client-side + // processing + if (server) { + // if a search string has been pre-set, search now + if (searchCol) $input.trigger('input').trigger('change'); + return; + } + + var customFilter = function(settings, data, dataIndex) { + // there is no way to attach a search function to a specific table, + // and we need to make sure a global search function is not applied to + // all tables (i.e. a range filter in a previous table should not be + // applied to the current table); we use the settings object to + // determine if we want to perform searching on the current table, + // since settings.sTableId will be different to different tables + if (table.settings()[0] !== settings) return true; + // no filter on this column or no need to filter this column + if (typeof filter === 'undefined' || !$td.data('filter')) return true; + + var r = filter.val(), v, r0, r1; + var i_data = function(i) { + if (!colReorderEnabled()) return i; + var order = table.colReorder.order(), k; + for (k = 0; k < order.length; ++k) if (order[k] === i) return k; + return i; // in theory it will never be here... + } + v = data[i_data(i)]; + if (type === 'number' || type === 'integer') { + v = parseFloat(v); + // how to handle NaN? currently exclude these rows + if (isNaN(v)) return(false); + r0 = parseFloat(scaleBack(r[0], scale)) + r1 = parseFloat(scaleBack(r[1], scale)); + if (v >= r0 && v <= r1) return true; + } else if (type === 'date' || type === 'time') { + v = new Date(v); + r0 = new Date(r[0] / scale); r1 = new Date(r[1] / scale); + if (v >= r0 && v <= r1) return true; + } else if (type === 'factor') { + if (r.length === 0 || inArray(v, r)) return true; + } else if (type === 'logical') { + if (r.length === 0) return true; + if (inArray(v === '' ? 'na' : v, r)) return true; + } + return false; + }; + + $.fn.dataTable.ext.search.push(customFilter); + + // search for the preset search strings if it is non-empty + if (searchCol) $input.trigger('input').trigger('change'); + + }); + + } + + // highlight search keywords + var highlight = function() { + var body = $(table.table().body()); + // removing the old highlighting first + body.unhighlight(); + + // don't highlight the "not found" row, so we get the rows using the api + if (table.rows({ filter: 'applied' }).data().length === 0) return; + // highlight global search keywords + body.highlight($.trim(table.search()).split(/\s+/)); + // then highlight keywords from individual column filters + if (filterRow) filterRow.each(function(i, td) { + var $td = $(td), type = $td.data('type'); + if (type !== 'character') return; + var $input = $td.children('div').first().children('input'); + var column = table.column(i).nodes().to$(), + val = $.trim($input.val()); + if (type !== 'character' || val === '') return; + column.highlight(val.split(/\s+/)); + }); + }; + + if (options.searchHighlight) { + table + .on('draw.dt.dth column-visibility.dt.dth column-reorder.dt.dth', highlight) + .on('destroy', function() { + // remove event handler + table.off('draw.dt.dth column-visibility.dt.dth column-reorder.dt.dth'); + }); + + // Set the option for escaping regex characters in our search string. This will be used + // for all future matching. + jQuery.fn.highlight.options.escapeRegex = (!options.search || !options.search.regex); + + // initial highlight for state saved conditions and initial states + highlight(); + } + + // run the callback function on the table instance + if (typeof data.callback === 'function') data.callback(table); + + // double click to edit the cell, row, column, or all cells + if (data.editable) table.on('dblclick.dt', 'tbody td', function(e) { + // only bring up the editor when the cell itself is dbclicked, and ignore + // other dbclick events bubbled up (e.g. from the ) + if (e.target !== this) return; + var target = [], immediate = false; + switch (data.editable.target) { + case 'cell': + target = [this]; + immediate = true; // edit will take effect immediately + break; + case 'row': + target = table.cells(table.cell(this).index().row, '*').nodes(); + break; + case 'column': + target = table.cells('*', table.cell(this).index().column).nodes(); + break; + case 'all': + target = table.cells().nodes(); + break; + default: + throw 'The editable parameter must be "cell", "row", "column", or "all"'; + } + var disableCols = data.editable.disable ? data.editable.disable.columns : null; + var numericCols = data.editable.numeric; + var areaCols = data.editable.area; + var dateCols = data.editable.date; + for (var i = 0; i < target.length; i++) { + (function(cell, current) { + var $cell = $(cell), html = $cell.html(); + var _cell = table.cell(cell), value = _cell.data(), index = _cell.index().column; + var $input; + if (inArray(index, numericCols)) { + $input = $(''); + } else if (inArray(index, areaCols)) { + $input = $(''); + } else if (inArray(index, dateCols)) { + $input = $(''); + } else { + $input = $(''); + } + if (!immediate) { + $cell.data('input', $input).data('html', html); + $input.attr('title', 'Hit Ctrl+Enter to finish editing, or Esc to cancel'); + } + $input.val(value); + if (inArray(index, disableCols)) { + $input.attr('readonly', '').css('filter', 'invert(25%)'); + } + $cell.empty().append($input); + if (cell === current) $input.focus(); + $input.css('width', '100%'); + + if (immediate) $input.on('blur', function(e) { + var valueNew = $input.val(); + if (valueNew !== value) { + _cell.data(valueNew); + if (HTMLWidgets.shinyMode) { + changeInput('cell_edit', [cellInfo(cell)], 'DT.cellInfo', null, {priority: 'event'}); + } + // for server-side processing, users have to call replaceData() to update the table + if (!server) table.draw(false); + } else { + $cell.html(html); + } + }).on('keyup', function(e) { + // hit Escape to cancel editing + if (e.keyCode === 27) $input.trigger('blur'); + }); + + // bulk edit (row, column, or all) + if (!immediate) $input.on('keyup', function(e) { + var removeInput = function($cell, restore) { + $cell.data('input').remove(); + if (restore) $cell.html($cell.data('html')); + } + if (e.keyCode === 27) { + for (var i = 0; i < target.length; i++) { + removeInput($(target[i]), true); + } + } else if (e.keyCode === 13 && e.ctrlKey) { + // Ctrl + Enter + var cell, $cell, _cell, cellData = []; + for (var i = 0; i < target.length; i++) { + cell = target[i]; $cell = $(cell); _cell = table.cell(cell); + _cell.data($cell.data('input').val()); + HTMLWidgets.shinyMode && cellData.push(cellInfo(cell)); + removeInput($cell, false); + } + if (HTMLWidgets.shinyMode) { + changeInput('cell_edit', cellData, 'DT.cellInfo', null, {priority: "event"}); + } + if (!server) table.draw(false); + } + }); + })(target[i], this); + } + }); + + // interaction with shiny + if (!HTMLWidgets.shinyMode && !crosstalkOptions.group) return; + + var methods = {}; + var shinyData = {}; + + methods.updateCaption = function(caption) { + if (!caption) return; + $table.children('caption').replaceWith(caption); + } + + // register clear functions to remove input values when the table is removed + instance.clearInputs = {}; + + var changeInput = function(id, value, type, noCrosstalk, opts) { + var event = id; + id = el.id + '_' + id; + if (type) id = id + ':' + type; + // do not update if the new value is the same as old value + if (event !== 'cell_edit' && !/_clicked$/.test(event) && shinyData.hasOwnProperty(id) && shinyData[id] === JSON.stringify(value)) + return; + shinyData[id] = JSON.stringify(value); + if (HTMLWidgets.shinyMode && Shiny.setInputValue) { + Shiny.setInputValue(id, value, opts); + if (!instance.clearInputs[id]) instance.clearInputs[id] = function() { + Shiny.setInputValue(id, null); + } + } + + // HACK + if (event === "rows_selected" && !noCrosstalk) { + if (crosstalkOptions.group) { + var keys = crosstalkOptions.key; + var selectedKeys = null; + if (value) { + selectedKeys = []; + for (var i = 0; i < value.length; i++) { + // The value array's contents use 1-based row numbers, so we must + // convert to 0-based before indexing into the keys array. + selectedKeys.push(keys[value[i] - 1]); + } + } + instance.ctselectHandle.set(selectedKeys); + } + } + }; + + var addOne = function(x) { + return x.map(function(i) { return 1 + i; }); + }; + + var unique = function(x) { + var ux = []; + $.each(x, function(i, el){ + if ($.inArray(el, ux) === -1) ux.push(el); + }); + return ux; + } + + // change the row index of a cell + var tweakCellIndex = function(cell) { + var info = cell.index(); + // some cell may not be valid. e.g, #759 + // when using the RowGroup extension, datatables will + // generate the row label and the cells are not part of + // the data thus contain no row/col info + if (info === undefined) + return {row: null, col: null}; + if (server) { + info.row = DT_rows_current[info.row]; + } else { + info.row += 1; + } + return {row: info.row, col: info.column}; + } + + var cleanSelectedValues = function() { + changeInput('rows_selected', []); + changeInput('columns_selected', []); + changeInput('cells_selected', transposeArray2D([]), 'shiny.matrix'); + } + // #828 we should clean the selection on the server-side when the table reloads + cleanSelectedValues(); + + // a flag to indicates if select extension is initialized or not + var flagSelectExt = table.settings()[0]._select !== undefined; + // the Select extension should only be used in the client mode and + // when the selection.mode is set to none + if (data.selection.mode === 'none' && !server && flagSelectExt) { + var updateRowsSelected = function() { + var rows = table.rows({selected: true}); + var selected = []; + $.each(rows.indexes().toArray(), function(i, v) { + selected.push(v + 1); + }); + changeInput('rows_selected', selected); + } + var updateColsSelected = function() { + var columns = table.columns({selected: true}); + changeInput('columns_selected', columns.indexes().toArray()); + } + var updateCellsSelected = function() { + var cells = table.cells({selected: true}); + var selected = []; + cells.every(function() { + var row = this.index().row; + var col = this.index().column; + selected = selected.concat([[row + 1, col]]); + }); + changeInput('cells_selected', transposeArray2D(selected), 'shiny.matrix'); + } + table.on('select deselect', function(e, dt, type, indexes) { + updateRowsSelected(); + updateColsSelected(); + updateCellsSelected(); + }) + updateRowsSelected(); + updateColsSelected(); + updateCellsSelected(); + } + + var selMode = data.selection.mode, selTarget = data.selection.target; + var selDisable = data.selection.selectable === false; + if (inArray(selMode, ['single', 'multiple'])) { + var selClass = inArray(data.style, ['bootstrap', 'bootstrap4']) ? 'active' : 'selected'; + // selected1: row indices; selected2: column indices + var initSel = function(x) { + if (x === null || typeof x === 'boolean' || selTarget === 'cell') { + return {rows: [], cols: []}; + } else if (selTarget === 'row') { + return {rows: $.makeArray(x), cols: []}; + } else if (selTarget === 'column') { + return {rows: [], cols: $.makeArray(x)}; + } else if (selTarget === 'row+column') { + return {rows: $.makeArray(x.rows), cols: $.makeArray(x.cols)}; + } + } + var selected = data.selection.selected; + var selected1 = initSel(selected).rows, selected2 = initSel(selected).cols; + // selectable should contain either all positive or all non-positive values, not both + // positive values indicate "selectable" while non-positive values means "nonselectable" + // the assertion is performed on R side. (only column indicides could be zero which indicates + // the row name) + var selectable = data.selection.selectable; + var selectable1 = initSel(selectable).rows, selectable2 = initSel(selectable).cols; + + // After users reorder the rows or filter the table, we cannot use the table index + // directly. Instead, we need this function to find out the rows between the two clicks. + // If user filter the table again between the start click and the end click, the behavior + // would be undefined, but it should not be a problem. + var shiftSelRowsIndex = function(start, end) { + var indexes = server ? DT_rows_all : table.rows({ search: 'applied' }).indexes().toArray(); + start = indexes.indexOf(start); end = indexes.indexOf(end); + // if start is larger than end, we need to swap + if (start > end) { + var tmp = end; end = start; start = tmp; + } + return indexes.slice(start, end + 1); + } + + var serverRowIndex = function(clientRowIndex) { + return server ? DT_rows_current[clientRowIndex] : clientRowIndex + 1; + } + + // row, column, or cell selection + var lastClickedRow; + if (inArray(selTarget, ['row', 'row+column'])) { + // Get the current selected rows. It will also + // update the selected1's value based on the current row selection state + // Note we can't put this function inside selectRows() directly, + // the reason is method.selectRows() will override selected1's value but this + // function will add rows to selected1 (keep the existing selection), which is + // inconsistent with column and cell selection. + var selectedRows = function() { + var rows = table.rows('.' + selClass); + var idx = rows.indexes().toArray(); + if (!server) { + selected1 = addOne(idx); + return selected1; + } + idx = idx.map(function(i) { + return DT_rows_current[i]; + }); + selected1 = selMode === 'multiple' ? unique(selected1.concat(idx)) : idx; + return selected1; + } + // Change selected1's value based on selectable1, then refresh the row state + var onlyKeepSelectableRows = function() { + if (selDisable) { // users can't select; useful when only want backend select + selected1 = []; + return; + } + if (selectable1.length === 0) return; + var nonselectable = selectable1[0] <= 0; + if (nonselectable) { + // should make selectable1 positive + selected1 = $(selected1).not(selectable1.map(function(i) { return -i; })).get(); + } else { + selected1 = $(selected1).filter(selectable1).get(); + } + } + // Change selected1's value based on selectable1, then + // refresh the row selection state according to values in selected1 + var selectRows = function(ignoreSelectable) { + if (!ignoreSelectable) onlyKeepSelectableRows(); + table.$('tr.' + selClass).removeClass(selClass); + if (selected1.length === 0) return; + if (server) { + table.rows({page: 'current'}).every(function() { + if (inArray(DT_rows_current[this.index()], selected1)) { + $(this.node()).addClass(selClass); + } + }); + } else { + var selected0 = selected1.map(function(i) { return i - 1; }); + $(table.rows(selected0).nodes()).addClass(selClass); + } + } + table.on('mousedown.dt', 'tbody tr', function(e) { + var $this = $(this), thisRow = table.row(this); + if (selMode === 'multiple') { + if (e.shiftKey && lastClickedRow !== undefined) { + // select or de-select depends on the last clicked row's status + var flagSel = !$this.hasClass(selClass); + var crtClickedRow = serverRowIndex(thisRow.index()); + if (server) { + var rowsIndex = shiftSelRowsIndex(lastClickedRow, crtClickedRow); + // update current page's selClass + rowsIndex.map(function(i) { + var rowIndex = DT_rows_current.indexOf(i); + if (rowIndex >= 0) { + var row = table.row(rowIndex).nodes().to$(); + var flagRowSel = !row.hasClass(selClass); + if (flagSel === flagRowSel) row.toggleClass(selClass); + } + }); + // update selected1 + if (flagSel) { + selected1 = unique(selected1.concat(rowsIndex)); + } else { + selected1 = selected1.filter(function(index) { + return !inArray(index, rowsIndex); + }); + } + } else { + // js starts from 0 + shiftSelRowsIndex(lastClickedRow - 1, crtClickedRow - 1).map(function(value) { + var row = table.row(value).nodes().to$(); + var flagRowSel = !row.hasClass(selClass); + if (flagSel === flagRowSel) row.toggleClass(selClass); + }); + } + e.preventDefault(); + } else { + $this.toggleClass(selClass); + } + } else { + if ($this.hasClass(selClass)) { + $this.removeClass(selClass); + } else { + table.$('tr.' + selClass).removeClass(selClass); + $this.addClass(selClass); + } + } + if (server && !$this.hasClass(selClass)) { + var id = DT_rows_current[thisRow.index()]; + // remove id from selected1 since its class .selected has been removed + if (inArray(id, selected1)) selected1.splice($.inArray(id, selected1), 1); + } + selectedRows(); // update selected1's value based on selClass + selectRows(false); // only keep the selectable rows + changeInput('rows_selected', selected1); + changeInput('row_last_clicked', serverRowIndex(thisRow.index()), null, null, {priority: 'event'}); + lastClickedRow = serverRowIndex(thisRow.index()); + }); + selectRows(false); // in case users have specified pre-selected rows + // restore selected rows after the table is redrawn (e.g. sort/search/page); + // client-side tables will preserve the selections automatically; for + // server-side tables, we have to *real* row indices are in `selected1` + changeInput('rows_selected', selected1); + if (server) table.on('draw.dt', function(e) { selectRows(false); }); + methods.selectRows = function(selected, ignoreSelectable) { + selected1 = $.makeArray(selected); + selectRows(ignoreSelectable); + changeInput('rows_selected', selected1); + } + } + + if (inArray(selTarget, ['column', 'row+column'])) { + if (selTarget === 'row+column') { + $(table.columns().footer()).css('cursor', 'pointer'); + } + // update selected2's value based on selectable2 + var onlyKeepSelectableCols = function() { + if (selDisable) { // users can't select; useful when only want backend select + selected2 = []; + return; + } + if (selectable2.length === 0) return; + var nonselectable = selectable2[0] <= 0; + if (nonselectable) { + // need to make selectable2 positive + selected2 = $(selected2).not(selectable2.map(function(i) { return -i; })).get(); + } else { + selected2 = $(selected2).filter(selectable2).get(); + } + } + // update selected2 and then + // refresh the col selection state according to values in selected2 + var selectCols = function(ignoreSelectable) { + if (!ignoreSelectable) onlyKeepSelectableCols(); + // if selected2 is not a valide index (e.g., larger than the column number) + // table.columns(selected2) will fail and result in a blank table + // this is different from the table.rows(), where the out-of-range indexes + // doesn't affect at all + selected2 = $(selected2).filter(table.columns().indexes()).get(); + table.columns().nodes().flatten().to$().removeClass(selClass); + if (selected2.length > 0) + table.columns(selected2).nodes().flatten().to$().addClass(selClass); + } + var callback = function() { + var colIdx = selTarget === 'column' ? table.cell(this).index().column : + $.inArray(this, table.columns().footer()), + thisCol = $(table.column(colIdx).nodes()); + if (colIdx === -1) return; + if (thisCol.hasClass(selClass)) { + thisCol.removeClass(selClass); + selected2.splice($.inArray(colIdx, selected2), 1); + } else { + if (selMode === 'single') $(table.cells().nodes()).removeClass(selClass); + thisCol.addClass(selClass); + selected2 = selMode === 'single' ? [colIdx] : unique(selected2.concat([colIdx])); + } + selectCols(false); // update selected2 based on selectable + changeInput('columns_selected', selected2); + } + if (selTarget === 'column') { + $(table.table().body()).on('click.dt', 'td', callback); + } else { + $(table.table().footer()).on('click.dt', 'tr th', callback); + } + selectCols(false); // in case users have specified pre-selected columns + changeInput('columns_selected', selected2); + if (server) table.on('draw.dt', function(e) { selectCols(false); }); + methods.selectColumns = function(selected, ignoreSelectable) { + selected2 = $.makeArray(selected); + selectCols(ignoreSelectable); + changeInput('columns_selected', selected2); + } + } + + if (selTarget === 'cell') { + var selected3 = [], selectable3 = []; + if (selected !== null) selected3 = selected; + if (selectable !== null && typeof selectable !== 'boolean') selectable3 = selectable; + var findIndex = function(ij, sel) { + for (var i = 0; i < sel.length; i++) { + if (ij[0] === sel[i][0] && ij[1] === sel[i][1]) return i; + } + return -1; + } + // Change selected3's value based on selectable3, then refresh the cell state + var onlyKeepSelectableCells = function() { + if (selDisable) { // users can't select; useful when only want backend select + selected3 = []; + return; + } + if (selectable3.length === 0) return; + var nonselectable = selectable3[0][0] <= 0; + var out = []; + if (nonselectable) { + selected3.map(function(ij) { + // should make selectable3 positive + if (findIndex([-ij[0], -ij[1]], selectable3) === -1) { out.push(ij); } + }); + } else { + selected3.map(function(ij) { + if (findIndex(ij, selectable3) > -1) { out.push(ij); } + }); + } + selected3 = out; + } + // Change selected3's value based on selectable3, then + // refresh the cell selection state according to values in selected3 + var selectCells = function(ignoreSelectable) { + if (!ignoreSelectable) onlyKeepSelectableCells(); + table.$('td.' + selClass).removeClass(selClass); + if (selected3.length === 0) return; + if (server) { + table.cells({page: 'current'}).every(function() { + var info = tweakCellIndex(this); + if (findIndex([info.row, info.col], selected3) > -1) + $(this.node()).addClass(selClass); + }); + } else { + selected3.map(function(ij) { + $(table.cell(ij[0] - 1, ij[1]).node()).addClass(selClass); + }); + } + }; + table.on('click.dt', 'tbody td', function() { + var $this = $(this), info = tweakCellIndex(table.cell(this)); + if ($this.hasClass(selClass)) { + $this.removeClass(selClass); + selected3.splice(findIndex([info.row, info.col], selected3), 1); + } else { + if (selMode === 'single') $(table.cells().nodes()).removeClass(selClass); + $this.addClass(selClass); + selected3 = selMode === 'single' ? [[info.row, info.col]] : + unique(selected3.concat([[info.row, info.col]])); + } + selectCells(false); // must call this to update selected3 based on selectable3 + changeInput('cells_selected', transposeArray2D(selected3), 'shiny.matrix'); + }); + selectCells(false); // in case users have specified pre-selected columns + changeInput('cells_selected', transposeArray2D(selected3), 'shiny.matrix'); + + if (server) table.on('draw.dt', function(e) { selectCells(false); }); + methods.selectCells = function(selected, ignoreSelectable) { + selected3 = selected ? selected : []; + selectCells(ignoreSelectable); + changeInput('cells_selected', transposeArray2D(selected3), 'shiny.matrix'); + } + } + } + + // expose some table info to Shiny + var updateTableInfo = function(e, settings) { + // TODO: is anyone interested in the page info? + // changeInput('page_info', table.page.info()); + var updateRowInfo = function(id, modifier) { + var idx; + if (server) { + idx = modifier.page === 'current' ? DT_rows_current : DT_rows_all; + } else { + var rows = table.rows($.extend({ + search: 'applied', + page: 'all' + }, modifier)); + idx = addOne(rows.indexes().toArray()); + } + changeInput('rows' + '_' + id, idx); + }; + updateRowInfo('current', {page: 'current'}); + updateRowInfo('all', {}); + } + table.on('draw.dt', updateTableInfo); + updateTableInfo(); + + // state info + table.on('draw.dt column-visibility.dt', function() { + changeInput('state', table.state()); + }); + changeInput('state', table.state()); + + // search info + var updateSearchInfo = function() { + changeInput('search', table.search()); + if (filterRow) changeInput('search_columns', filterRow.toArray().map(function(td) { + return $(td).find('input').first().val(); + })); + } + table.on('draw.dt', updateSearchInfo); + updateSearchInfo(); + + var cellInfo = function(thiz) { + var info = tweakCellIndex(table.cell(thiz)); + info.value = table.cell(thiz).data(); + return info; + } + // the current cell clicked on + table.on('click.dt', 'tbody td', function() { + changeInput('cell_clicked', cellInfo(this), null, null, {priority: 'event'}); + }) + changeInput('cell_clicked', {}); + + // do not trigger table selection when clicking on links unless they have classes + table.on('mousedown.dt', 'tbody td a', function(e) { + if (this.className === '') e.stopPropagation(); + }); + + methods.addRow = function(data, rowname, resetPaging) { + var n = table.columns().indexes().length, d = n - data.length; + if (d === 1) { + data = rowname.concat(data) + } else if (d !== 0) { + console.log(data); + console.log(table.columns().indexes()); + throw 'New data must be of the same length as current data (' + n + ')'; + }; + table.row.add(data).draw(resetPaging); + } + + methods.updateSearch = function(keywords) { + if (keywords.global !== null) + $(table.table().container()).find('input[type=search]').first() + .val(keywords.global).trigger('input'); + var columns = keywords.columns; + if (!filterRow || columns === null) return; + filterRow.toArray().map(function(td, i) { + var v = typeof columns === 'string' ? columns : columns[i]; + if (typeof v === 'undefined') { + console.log('The search keyword for column ' + i + ' is undefined') + return; + } + // Update column search string and values on linked filter widgets. + // 'input' for factor and char filters, 'change' for numeric filters. + $(td).find('input').first().val(v).trigger('input', [true]).trigger('change'); + }); + table.draw(); + } + + methods.hideCols = function(hide, reset) { + if (reset) table.columns().visible(true, false); + table.columns(hide).visible(false); + } + + methods.showCols = function(show, reset) { + if (reset) table.columns().visible(false, false); + table.columns(show).visible(true); + } + + methods.colReorder = function(order, origOrder) { + table.colReorder.order(order, origOrder); + } + + methods.selectPage = function(page) { + if (table.page.info().pages < page || page < 1) { + throw 'Selected page is out of range'; + }; + table.page(page - 1).draw(false); + } + + methods.reloadData = function(resetPaging, clearSelection) { + // empty selections first if necessary + if (methods.selectRows && inArray('row', clearSelection)) methods.selectRows([]); + if (methods.selectColumns && inArray('column', clearSelection)) methods.selectColumns([]); + if (methods.selectCells && inArray('cell', clearSelection)) methods.selectCells([]); + table.ajax.reload(null, resetPaging); + } + + // update table filters (set new limits of sliders) + methods.updateFilters = function(newProps) { + // loop through each filter in the filter row + filterRow.each(function(i, td) { + var k = i; + if (filterRow.length > newProps.length) { + if (i === 0) return; // first column is row names + k = i - 1; + } + // Update the filters to reflect the updated data. + // Allow "falsy" (e.g. NULL) to signify a no-op. + if (newProps[k]) { + setFilterProps(td, newProps[k]); + } + }); + }; + + table.shinyMethods = methods; + }, + resize: function(el, width, height, instance) { + if (instance.data) this.renderValue(el, instance.data, instance); + + // dynamically adjust height if fillContainer = TRUE + if (instance.fillContainer) + this.fillAvailableHeight(el, height); + + this.adjustWidth(el); + }, + + // dynamically set the scroll body to fill available height + // (used with fillContainer = TRUE) + fillAvailableHeight: function(el, availableHeight) { + + // see how much of the table is occupied by header/footer elements + // and use that to compute a target scroll body height + var dtWrapper = $(el).find('div.dataTables_wrapper'); + var dtScrollBody = $(el).find($('div.dataTables_scrollBody')); + var framingHeight = dtWrapper.innerHeight() - dtScrollBody.innerHeight(); + var scrollBodyHeight = availableHeight - framingHeight; + + // we need to set `max-height` to none as datatables library now sets this + // to a fixed height, disabling the ability to resize to fill the window, + // as it will be set to a fixed 100px under such circumstances, e.g., RStudio IDE, + // or FlexDashboard + // see https://github.com/rstudio/DT/issues/951#issuecomment-1026464509 + dtScrollBody.css('max-height', 'none'); + // set the height + dtScrollBody.height(scrollBodyHeight + 'px'); + }, + + // adjust the width of columns; remove the hard-coded widths on table and the + // scroll header when scrollX/Y are enabled + adjustWidth: function(el) { + var $el = $(el), table = $el.data('datatable'); + if (table) table.columns.adjust(); + $el.find('.dataTables_scrollHeadInner').css('width', '') + .children('table').css('margin-left', ''); + } +}); + + if (!HTMLWidgets.shinyMode) return; + + Shiny.addCustomMessageHandler('datatable-calls', function(data) { + var id = data.id; + var el = document.getElementById(id); + var table = el ? $(el).data('datatable') : null; + if (!table) { + console.log("Couldn't find table with id " + id); + return; + } + + var methods = table.shinyMethods, call = data.call; + if (methods[call.method]) { + methods[call.method].apply(table, call.args); + } else { + console.log("Unknown method " + call.method); + } + }); + +})(); diff --git a/_freeze/site_libs/datatables-css-0.0.0/datatables-crosstalk.css b/_freeze/site_libs/datatables-css-0.0.0/datatables-crosstalk.css new file mode 100644 index 00000000..bd1159c8 --- /dev/null +++ b/_freeze/site_libs/datatables-css-0.0.0/datatables-crosstalk.css @@ -0,0 +1,32 @@ +.dt-crosstalk-fade { + opacity: 0.2; +} + +html body div.DTS div.dataTables_scrollBody { + background: none; +} + + +/* +Fix https://github.com/rstudio/DT/issues/563 +If the `table.display` is set to "block" (e.g., pkgdown), the browser will display +datatable objects strangely. The search panel and the page buttons will still be +in full-width but the table body will be "compact" and shorter. +In therory, having this attributes will affect `dom="t"` +with `display: block` users. But in reality, there should be no one. +We may remove the below lines in the future if the upstream agree to have this there. +See https://github.com/DataTables/DataTablesSrc/issues/160 +*/ + +table.dataTable { + display: table; +} + + +/* +When DTOutput(fill = TRUE), it receives a .html-fill-item class (via htmltools::bindFillRole()), which effectively amounts to `flex: 1 1 auto`. That's mostly fine, but the case where `fillContainer=TRUE`+`height:auto`+`flex-basis:auto` and the container (e.g., a bslib::card()) doesn't have a defined height is a bit problematic since the table wants to fit the parent but the parent wants to fit the table, which results pretty small table height (maybe because there is a minimum height somewhere?). It seems better in this case to impose a 400px height default for the table, which we can do by setting `flex-basis` to 400px (the table is still allowed to grow/shrink when the container has an opinionated height). +*/ + +.html-fill-container > .html-fill-item.datatables { + flex-basis: 400px; +} diff --git a/_freeze/site_libs/dt-core-1.13.6/css/jquery.dataTables.extra.css b/_freeze/site_libs/dt-core-1.13.6/css/jquery.dataTables.extra.css new file mode 100644 index 00000000..b2dd141f --- /dev/null +++ b/_freeze/site_libs/dt-core-1.13.6/css/jquery.dataTables.extra.css @@ -0,0 +1,28 @@ +/* Selected rows/cells */ +table.dataTable tr.selected td, table.dataTable td.selected { + background-color: #b0bed9 !important; +} +/* In case of scrollX/Y or FixedHeader */ +.dataTables_scrollBody .dataTables_sizing { + visibility: hidden; +} + +/* The datatables' theme CSS file doesn't define +the color but with white background. It leads to an issue that +when the HTML's body color is set to 'white', the user can't +see the text since the background is white. One case happens in the +RStudio's IDE when inline viewing the DT table inside an Rmd file, +if the IDE theme is set to "Cobalt". + +See https://github.com/rstudio/DT/issues/447 for more info + +This fixes should have little side-effects because all the other elements +of the default theme use the #333 font color. + +TODO: The upstream may use relative colors for both the table background +and the color. It means the table can display well without this patch +then. At that time, we need to remove the below CSS attributes. +*/ +div.datatables { + color: #333; +} diff --git a/_freeze/site_libs/dt-core-1.13.6/css/jquery.dataTables.min.css b/_freeze/site_libs/dt-core-1.13.6/css/jquery.dataTables.min.css new file mode 100644 index 00000000..ad59f843 --- /dev/null +++ b/_freeze/site_libs/dt-core-1.13.6/css/jquery.dataTables.min.css @@ -0,0 +1 @@ +:root{--dt-row-selected: 13, 110, 253;--dt-row-selected-text: 255, 255, 255;--dt-row-selected-link: 9, 10, 11;--dt-row-stripe: 0, 0, 0;--dt-row-hover: 0, 0, 0;--dt-column-ordering: 0, 0, 0;--dt-html-background: white}:root.dark{--dt-html-background: rgb(33, 37, 41)}table.dataTable td.dt-control{text-align:center;cursor:pointer}table.dataTable td.dt-control:before{display:inline-block;color:rgba(0, 0, 0, 0.5);content:"►"}table.dataTable tr.dt-hasChild td.dt-control:before{content:"▼"}html.dark table.dataTable td.dt-control:before{color:rgba(255, 255, 255, 0.5)}html.dark table.dataTable tr.dt-hasChild td.dt-control:before{color:rgba(255, 255, 255, 0.5)}table.dataTable thead>tr>th.sorting,table.dataTable thead>tr>th.sorting_asc,table.dataTable thead>tr>th.sorting_desc,table.dataTable thead>tr>th.sorting_asc_disabled,table.dataTable thead>tr>th.sorting_desc_disabled,table.dataTable thead>tr>td.sorting,table.dataTable thead>tr>td.sorting_asc,table.dataTable thead>tr>td.sorting_desc,table.dataTable thead>tr>td.sorting_asc_disabled,table.dataTable thead>tr>td.sorting_desc_disabled{cursor:pointer;position:relative;padding-right:26px}table.dataTable thead>tr>th.sorting:before,table.dataTable thead>tr>th.sorting:after,table.dataTable thead>tr>th.sorting_asc:before,table.dataTable thead>tr>th.sorting_asc:after,table.dataTable thead>tr>th.sorting_desc:before,table.dataTable thead>tr>th.sorting_desc:after,table.dataTable thead>tr>th.sorting_asc_disabled:before,table.dataTable thead>tr>th.sorting_asc_disabled:after,table.dataTable thead>tr>th.sorting_desc_disabled:before,table.dataTable thead>tr>th.sorting_desc_disabled:after,table.dataTable thead>tr>td.sorting:before,table.dataTable thead>tr>td.sorting:after,table.dataTable thead>tr>td.sorting_asc:before,table.dataTable thead>tr>td.sorting_asc:after,table.dataTable thead>tr>td.sorting_desc:before,table.dataTable thead>tr>td.sorting_desc:after,table.dataTable thead>tr>td.sorting_asc_disabled:before,table.dataTable thead>tr>td.sorting_asc_disabled:after,table.dataTable thead>tr>td.sorting_desc_disabled:before,table.dataTable thead>tr>td.sorting_desc_disabled:after{position:absolute;display:block;opacity:.125;right:10px;line-height:9px;font-size:.8em}table.dataTable thead>tr>th.sorting:before,table.dataTable thead>tr>th.sorting_asc:before,table.dataTable thead>tr>th.sorting_desc:before,table.dataTable thead>tr>th.sorting_asc_disabled:before,table.dataTable thead>tr>th.sorting_desc_disabled:before,table.dataTable thead>tr>td.sorting:before,table.dataTable thead>tr>td.sorting_asc:before,table.dataTable thead>tr>td.sorting_desc:before,table.dataTable thead>tr>td.sorting_asc_disabled:before,table.dataTable thead>tr>td.sorting_desc_disabled:before{bottom:50%;content:"▲";content:"▲"/""}table.dataTable thead>tr>th.sorting:after,table.dataTable thead>tr>th.sorting_asc:after,table.dataTable thead>tr>th.sorting_desc:after,table.dataTable thead>tr>th.sorting_asc_disabled:after,table.dataTable thead>tr>th.sorting_desc_disabled:after,table.dataTable thead>tr>td.sorting:after,table.dataTable thead>tr>td.sorting_asc:after,table.dataTable thead>tr>td.sorting_desc:after,table.dataTable thead>tr>td.sorting_asc_disabled:after,table.dataTable thead>tr>td.sorting_desc_disabled:after{top:50%;content:"▼";content:"▼"/""}table.dataTable thead>tr>th.sorting_asc:before,table.dataTable thead>tr>th.sorting_desc:after,table.dataTable thead>tr>td.sorting_asc:before,table.dataTable thead>tr>td.sorting_desc:after{opacity:.6}table.dataTable thead>tr>th.sorting_desc_disabled:after,table.dataTable thead>tr>th.sorting_asc_disabled:before,table.dataTable thead>tr>td.sorting_desc_disabled:after,table.dataTable thead>tr>td.sorting_asc_disabled:before{display:none}table.dataTable thead>tr>th:active,table.dataTable thead>tr>td:active{outline:none}div.dataTables_scrollBody>table.dataTable>thead>tr>th:before,div.dataTables_scrollBody>table.dataTable>thead>tr>th:after,div.dataTables_scrollBody>table.dataTable>thead>tr>td:before,div.dataTables_scrollBody>table.dataTable>thead>tr>td:after{display:none}div.dataTables_processing{position:absolute;top:50%;left:50%;width:200px;margin-left:-100px;margin-top:-26px;text-align:center;padding:2px}div.dataTables_processing>div:last-child{position:relative;width:80px;height:15px;margin:1em auto}div.dataTables_processing>div:last-child>div{position:absolute;top:0;width:13px;height:13px;border-radius:50%;background:rgb(13, 110, 253);background:rgb(var(--dt-row-selected));animation-timing-function:cubic-bezier(0, 1, 1, 0)}div.dataTables_processing>div:last-child>div:nth-child(1){left:8px;animation:datatables-loader-1 .6s infinite}div.dataTables_processing>div:last-child>div:nth-child(2){left:8px;animation:datatables-loader-2 .6s infinite}div.dataTables_processing>div:last-child>div:nth-child(3){left:32px;animation:datatables-loader-2 .6s infinite}div.dataTables_processing>div:last-child>div:nth-child(4){left:56px;animation:datatables-loader-3 .6s infinite}@keyframes datatables-loader-1{0%{transform:scale(0)}100%{transform:scale(1)}}@keyframes datatables-loader-3{0%{transform:scale(1)}100%{transform:scale(0)}}@keyframes datatables-loader-2{0%{transform:translate(0, 0)}100%{transform:translate(24px, 0)}}table.dataTable.nowrap th,table.dataTable.nowrap td{white-space:nowrap}table.dataTable th.dt-left,table.dataTable td.dt-left{text-align:left}table.dataTable th.dt-center,table.dataTable td.dt-center,table.dataTable td.dataTables_empty{text-align:center}table.dataTable th.dt-right,table.dataTable td.dt-right{text-align:right}table.dataTable th.dt-justify,table.dataTable td.dt-justify{text-align:justify}table.dataTable th.dt-nowrap,table.dataTable td.dt-nowrap{white-space:nowrap}table.dataTable thead th,table.dataTable thead td,table.dataTable tfoot th,table.dataTable tfoot td{text-align:left}table.dataTable thead th.dt-head-left,table.dataTable thead td.dt-head-left,table.dataTable tfoot th.dt-head-left,table.dataTable tfoot td.dt-head-left{text-align:left}table.dataTable thead th.dt-head-center,table.dataTable thead td.dt-head-center,table.dataTable tfoot th.dt-head-center,table.dataTable tfoot td.dt-head-center{text-align:center}table.dataTable thead th.dt-head-right,table.dataTable thead td.dt-head-right,table.dataTable tfoot th.dt-head-right,table.dataTable tfoot td.dt-head-right{text-align:right}table.dataTable thead th.dt-head-justify,table.dataTable thead td.dt-head-justify,table.dataTable tfoot th.dt-head-justify,table.dataTable tfoot td.dt-head-justify{text-align:justify}table.dataTable thead th.dt-head-nowrap,table.dataTable thead td.dt-head-nowrap,table.dataTable tfoot th.dt-head-nowrap,table.dataTable tfoot td.dt-head-nowrap{white-space:nowrap}table.dataTable tbody th.dt-body-left,table.dataTable tbody td.dt-body-left{text-align:left}table.dataTable tbody th.dt-body-center,table.dataTable tbody td.dt-body-center{text-align:center}table.dataTable tbody th.dt-body-right,table.dataTable tbody td.dt-body-right{text-align:right}table.dataTable tbody th.dt-body-justify,table.dataTable tbody td.dt-body-justify{text-align:justify}table.dataTable tbody th.dt-body-nowrap,table.dataTable tbody td.dt-body-nowrap{white-space:nowrap}table.dataTable{width:100%;margin:0 auto;clear:both;border-collapse:separate;border-spacing:0}table.dataTable thead th,table.dataTable tfoot th{font-weight:bold}table.dataTable>thead>tr>th,table.dataTable>thead>tr>td{padding:10px;border-bottom:1px solid rgba(0, 0, 0, 0.3)}table.dataTable>thead>tr>th:active,table.dataTable>thead>tr>td:active{outline:none}table.dataTable>tfoot>tr>th,table.dataTable>tfoot>tr>td{padding:10px 10px 6px 10px;border-top:1px solid rgba(0, 0, 0, 0.3)}table.dataTable tbody tr{background-color:transparent}table.dataTable tbody tr.selected>*{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.9);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.9);color:rgb(255, 255, 255);color:rgb(var(--dt-row-selected-text))}table.dataTable tbody tr.selected a{color:rgb(9, 10, 11);color:rgb(var(--dt-row-selected-link))}table.dataTable tbody th,table.dataTable tbody td{padding:8px 10px}table.dataTable.row-border>tbody>tr>th,table.dataTable.row-border>tbody>tr>td,table.dataTable.display>tbody>tr>th,table.dataTable.display>tbody>tr>td{border-top:1px solid rgba(0, 0, 0, 0.15)}table.dataTable.row-border>tbody>tr:first-child>th,table.dataTable.row-border>tbody>tr:first-child>td,table.dataTable.display>tbody>tr:first-child>th,table.dataTable.display>tbody>tr:first-child>td{border-top:none}table.dataTable.row-border>tbody>tr.selected+tr.selected>td,table.dataTable.display>tbody>tr.selected+tr.selected>td{border-top-color:#0262ef}table.dataTable.cell-border>tbody>tr>th,table.dataTable.cell-border>tbody>tr>td{border-top:1px solid rgba(0, 0, 0, 0.15);border-right:1px solid rgba(0, 0, 0, 0.15)}table.dataTable.cell-border>tbody>tr>th:first-child,table.dataTable.cell-border>tbody>tr>td:first-child{border-left:1px solid rgba(0, 0, 0, 0.15)}table.dataTable.cell-border>tbody>tr:first-child>th,table.dataTable.cell-border>tbody>tr:first-child>td{border-top:none}table.dataTable.stripe>tbody>tr.odd>*,table.dataTable.display>tbody>tr.odd>*{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.023);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-stripe), 0.023)}table.dataTable.stripe>tbody>tr.odd.selected>*,table.dataTable.display>tbody>tr.odd.selected>*{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.923);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.923)}table.dataTable.hover>tbody>tr:hover>*,table.dataTable.display>tbody>tr:hover>*{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.035);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.035)}table.dataTable.hover>tbody>tr.selected:hover>*,table.dataTable.display>tbody>tr.selected:hover>*{box-shadow:inset 0 0 0 9999px #0d6efd !important;box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 1) !important}table.dataTable.order-column>tbody tr>.sorting_1,table.dataTable.order-column>tbody tr>.sorting_2,table.dataTable.order-column>tbody tr>.sorting_3,table.dataTable.display>tbody tr>.sorting_1,table.dataTable.display>tbody tr>.sorting_2,table.dataTable.display>tbody tr>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.019);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.019)}table.dataTable.order-column>tbody tr.selected>.sorting_1,table.dataTable.order-column>tbody tr.selected>.sorting_2,table.dataTable.order-column>tbody tr.selected>.sorting_3,table.dataTable.display>tbody tr.selected>.sorting_1,table.dataTable.display>tbody tr.selected>.sorting_2,table.dataTable.display>tbody tr.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.919);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.919)}table.dataTable.display>tbody>tr.odd>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.odd>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.054);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.054)}table.dataTable.display>tbody>tr.odd>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.odd>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.047);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.047)}table.dataTable.display>tbody>tr.odd>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.odd>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.039);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.039)}table.dataTable.display>tbody>tr.odd.selected>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.odd.selected>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.954);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.954)}table.dataTable.display>tbody>tr.odd.selected>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.odd.selected>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.947);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.947)}table.dataTable.display>tbody>tr.odd.selected>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.odd.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.939);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.939)}table.dataTable.display>tbody>tr.even>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.even>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.019);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.019)}table.dataTable.display>tbody>tr.even>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.even>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.011);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.011)}table.dataTable.display>tbody>tr.even>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.even>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.003);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.003)}table.dataTable.display>tbody>tr.even.selected>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.even.selected>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.919);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.919)}table.dataTable.display>tbody>tr.even.selected>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.even.selected>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.911);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.911)}table.dataTable.display>tbody>tr.even.selected>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.even.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.903);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.903)}table.dataTable.display tbody tr:hover>.sorting_1,table.dataTable.order-column.hover tbody tr:hover>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.082);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.082)}table.dataTable.display tbody tr:hover>.sorting_2,table.dataTable.order-column.hover tbody tr:hover>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.074);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.074)}table.dataTable.display tbody tr:hover>.sorting_3,table.dataTable.order-column.hover tbody tr:hover>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.062);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.062)}table.dataTable.display tbody tr:hover.selected>.sorting_1,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.982);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.982)}table.dataTable.display tbody tr:hover.selected>.sorting_2,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.974);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.974)}table.dataTable.display tbody tr:hover.selected>.sorting_3,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.962);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.962)}table.dataTable.no-footer{border-bottom:1px solid rgba(0, 0, 0, 0.3)}table.dataTable.compact thead th,table.dataTable.compact thead td,table.dataTable.compact tfoot th,table.dataTable.compact tfoot td,table.dataTable.compact tbody th,table.dataTable.compact tbody td{padding:4px}table.dataTable th,table.dataTable td{box-sizing:content-box}.dataTables_wrapper{position:relative;clear:both}.dataTables_wrapper .dataTables_length{float:left}.dataTables_wrapper .dataTables_length select{border:1px solid #aaa;border-radius:3px;padding:5px;background-color:transparent;color:inherit;padding:4px}.dataTables_wrapper .dataTables_filter{float:right;text-align:right}.dataTables_wrapper .dataTables_filter input{border:1px solid #aaa;border-radius:3px;padding:5px;background-color:transparent;color:inherit;margin-left:3px}.dataTables_wrapper .dataTables_info{clear:both;float:left;padding-top:.755em}.dataTables_wrapper .dataTables_paginate{float:right;text-align:right;padding-top:.25em}.dataTables_wrapper .dataTables_paginate .paginate_button{box-sizing:border-box;display:inline-block;min-width:1.5em;padding:.5em 1em;margin-left:2px;text-align:center;text-decoration:none !important;cursor:pointer;color:inherit !important;border:1px solid transparent;border-radius:2px;background:transparent}.dataTables_wrapper .dataTables_paginate .paginate_button.current,.dataTables_wrapper .dataTables_paginate .paginate_button.current:hover{color:inherit !important;border:1px solid rgba(0, 0, 0, 0.3);background-color:rgba(0, 0, 0, 0.05);background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, rgba(230, 230, 230, 0.05)), color-stop(100%, rgba(0, 0, 0, 0.05)));background:-webkit-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:-moz-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:-ms-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:-o-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:linear-gradient(to bottom, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button.disabled,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active{cursor:default;color:#666 !important;border:1px solid transparent;background:transparent;box-shadow:none}.dataTables_wrapper .dataTables_paginate .paginate_button:hover{color:white !important;border:1px solid #111;background-color:#111;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111));background:-webkit-linear-gradient(top, #585858 0%, #111 100%);background:-moz-linear-gradient(top, #585858 0%, #111 100%);background:-ms-linear-gradient(top, #585858 0%, #111 100%);background:-o-linear-gradient(top, #585858 0%, #111 100%);background:linear-gradient(to bottom, #585858 0%, #111 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button:active{outline:none;background-color:#0c0c0c;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c));background:-webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%);box-shadow:inset 0 0 3px #111}.dataTables_wrapper .dataTables_paginate .ellipsis{padding:0 1em}.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter,.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_processing,.dataTables_wrapper .dataTables_paginate{color:inherit}.dataTables_wrapper .dataTables_scroll{clear:both}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody{-webkit-overflow-scrolling:touch}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>th,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>td,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>th,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>td{vertical-align:middle}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>th>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>td>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>th>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>td>div.dataTables_sizing{height:0;overflow:hidden;margin:0 !important;padding:0 !important}.dataTables_wrapper.no-footer .dataTables_scrollBody{border-bottom:1px solid rgba(0, 0, 0, 0.3)}.dataTables_wrapper.no-footer div.dataTables_scrollHead table.dataTable,.dataTables_wrapper.no-footer div.dataTables_scrollBody>table{border-bottom:none}.dataTables_wrapper:after{visibility:hidden;display:block;content:"";clear:both;height:0}@media screen and (max-width: 767px){.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_paginate{float:none;text-align:center}.dataTables_wrapper .dataTables_paginate{margin-top:.5em}}@media screen and (max-width: 640px){.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter{float:none;text-align:center}.dataTables_wrapper .dataTables_filter{margin-top:.5em}}html.dark{--dt-row-hover: 255, 255, 255;--dt-row-stripe: 255, 255, 255;--dt-column-ordering: 255, 255, 255}html.dark table.dataTable>thead>tr>th,html.dark table.dataTable>thead>tr>td{border-bottom:1px solid rgb(89, 91, 94)}html.dark table.dataTable>thead>tr>th:active,html.dark table.dataTable>thead>tr>td:active{outline:none}html.dark table.dataTable>tfoot>tr>th,html.dark table.dataTable>tfoot>tr>td{border-top:1px solid rgb(89, 91, 94)}html.dark table.dataTable.row-border>tbody>tr>th,html.dark table.dataTable.row-border>tbody>tr>td,html.dark table.dataTable.display>tbody>tr>th,html.dark table.dataTable.display>tbody>tr>td{border-top:1px solid rgb(64, 67, 70)}html.dark table.dataTable.row-border>tbody>tr.selected+tr.selected>td,html.dark table.dataTable.display>tbody>tr.selected+tr.selected>td{border-top-color:#0257d5}html.dark table.dataTable.cell-border>tbody>tr>th,html.dark table.dataTable.cell-border>tbody>tr>td{border-top:1px solid rgb(64, 67, 70);border-right:1px solid rgb(64, 67, 70)}html.dark table.dataTable.cell-border>tbody>tr>th:first-child,html.dark table.dataTable.cell-border>tbody>tr>td:first-child{border-left:1px solid rgb(64, 67, 70)}html.dark .dataTables_wrapper .dataTables_filter input,html.dark .dataTables_wrapper .dataTables_length select{border:1px solid rgba(255, 255, 255, 0.2);background-color:var(--dt-html-background)}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.current,html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.current:hover{border:1px solid rgb(89, 91, 94);background:rgba(255, 255, 255, 0.15)}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.disabled,html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover,html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active{color:#666 !important}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button:hover{border:1px solid rgb(53, 53, 53);background:rgb(53, 53, 53)}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button:active{background:#3a3a3a} diff --git a/_freeze/site_libs/dt-core-1.13.6/js/jquery.dataTables.min.js b/_freeze/site_libs/dt-core-1.13.6/js/jquery.dataTables.min.js new file mode 100644 index 00000000..f786b0da --- /dev/null +++ b/_freeze/site_libs/dt-core-1.13.6/js/jquery.dataTables.min.js @@ -0,0 +1,4 @@ +/*! DataTables 1.13.6 + * ©2008-2023 SpryMedia Ltd - datatables.net/license + */ +!function(n){"use strict";var a;"function"==typeof define&&define.amd?define(["jquery"],function(t){return n(t,window,document)}):"object"==typeof exports?(a=require("jquery"),"undefined"==typeof window?module.exports=function(t,e){return t=t||window,e=e||a(t),n(e,t,t.document)}:n(a,window,window.document)):window.DataTable=n(jQuery,window,document)}(function(P,j,v,H){"use strict";function d(t){var e=parseInt(t,10);return!isNaN(e)&&isFinite(t)?e:null}function l(t,e,n){var a=typeof t,r="string"==a;return"number"==a||"bigint"==a||!!h(t)||(e&&r&&(t=$(t,e)),n&&r&&(t=t.replace(q,"")),!isNaN(parseFloat(t))&&isFinite(t))}function a(t,e,n){var a;return!!h(t)||(h(a=t)||"string"==typeof a)&&!!l(t.replace(V,"").replace(/ - + diff --git a/docs/archive/2025-08-nyr/slides/advanced-01-introduction.html b/docs/archive/2025-08-nyr/slides/advanced-01-introduction.html index 6145d754..b46538c8 100644 --- a/docs/archive/2025-08-nyr/slides/advanced-01-introduction.html +++ b/docs/archive/2025-08-nyr/slides/advanced-01-introduction.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 1 - Introduction @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -894,190 +895,172 @@

Our versions

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/advanced-02-feature-engineering.html b/docs/archive/2025-08-nyr/slides/advanced-02-feature-engineering.html index 505717be..9500158b 100644 --- a/docs/archive/2025-08-nyr/slides/advanced-02-feature-engineering.html +++ b/docs/archive/2025-08-nyr/slides/advanced-02-feature-engineering.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 2 - Feature Engineering @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -211,13 +212,11 @@

2 - Feature Engineering

Working with our predictors

We might want to modify our predictors columns for a few reasons:

-
  • The model requires them in a different format (e.g. dummy variables for linear regression).
  • The model needs certain data qualities (e.g. same units for K-NN).
  • The outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).
-

The first two reasons are fairly predictable (next page).

The last one depends on your modeling problem.

@@ -1184,190 +1183,172 @@

More on recipes

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html b/docs/archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html index 21238307..1d5c21b4 100644 --- a/docs/archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html +++ b/docs/archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 3 - Tuning Hyperparameters @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -1164,190 +1165,172 @@

Your turn

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/advanced-04-racing.html b/docs/archive/2025-08-nyr/slides/advanced-04-racing.html index 37f298e5..ebdbf559 100644 --- a/docs/archive/2025-08-nyr/slides/advanced-04-racing.html +++ b/docs/archive/2025-08-nyr/slides/advanced-04-racing.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 4 - Grid Search via Racing @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -727,190 +728,172 @@

Your turn

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/advanced-05-iterative.html b/docs/archive/2025-08-nyr/slides/advanced-05-iterative.html index dc34d7bb..076dfa1a 100644 --- a/docs/archive/2025-08-nyr/slides/advanced-05-iterative.html +++ b/docs/archive/2025-08-nyr/slides/advanced-05-iterative.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 5 - Iterative Search @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -1075,190 +1076,172 @@

Test Set Results

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/advanced-06-wrapping-up.html b/docs/archive/2025-08-nyr/slides/advanced-06-wrapping-up.html index c335aabc..d50cb164 100644 --- a/docs/archive/2025-08-nyr/slides/advanced-06-wrapping-up.html +++ b/docs/archive/2025-08-nyr/slides/advanced-06-wrapping-up.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 6 - Wrapping up @@ -29,7 +29,7 @@ vertical-align: middle; } - + @@ -450,190 +450,172 @@

Resources to keep learning

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/annotations.html b/docs/archive/2025-08-nyr/slides/annotations.html index 2358b6cf..d78ef290 100644 --- a/docs/archive/2025-08-nyr/slides/annotations.html +++ b/docs/archive/2025-08-nyr/slides/annotations.html @@ -2,7 +2,7 @@ - + @@ -21,8 +21,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ +html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } -pre > code.sourceCode > span { line-height: 1.25; } +pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -33,7 +34,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } -pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } +pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -64,15 +65,16 @@ - + + - + - + + }); +
diff --git a/docs/archive/2025-08-nyr/slides/extras-effect-encodings.html b/docs/archive/2025-08-nyr/slides/extras-effect-encodings.html index d2915b76..6997c920 100644 --- a/docs/archive/2025-08-nyr/slides/extras-effect-encodings.html +++ b/docs/archive/2025-08-nyr/slides/extras-effect-encodings.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – Extras - Effect Encodings @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -700,190 +701,172 @@

Effect encoding results + }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/extras-transit-case-study.html b/docs/archive/2025-08-nyr/slides/extras-transit-case-study.html index d320986c..4a895aa6 100644 --- a/docs/archive/2025-08-nyr/slides/extras-transit-case-study.html +++ b/docs/archive/2025-08-nyr/slides/extras-transit-case-study.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – Case Study on Transportation @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -1087,190 +1088,172 @@

Plot the test set - window.document.addEventListener("DOMContentLoaded", function (event) { - const toggleBodyColorMode = (bsSheetEl) => { - const mode = bsSheetEl.getAttribute("data-mode"); - const bodyEl = window.document.querySelector("body"); - if (mode === "dark") { - bodyEl.classList.add("quarto-dark"); - bodyEl.classList.remove("quarto-light"); - } else { - bodyEl.classList.add("quarto-light"); - bodyEl.classList.remove("quarto-dark"); - } - } - const toggleBodyColorPrimary = () => { - const bsSheetEl = window.document.querySelector("link#quarto-bootstrap"); - if (bsSheetEl) { - toggleBodyColorMode(bsSheetEl); - } - } - toggleBodyColorPrimary(); - const tabsets = window.document.querySelectorAll(".panel-tabset-tabby") - tabsets.forEach(function(tabset) { - const tabby = new Tabby('#' + tabset.id); - }); - const isCodeAnnotation = (el) => { - for (const clz of el.classList) { - if (clz.startsWith('code-annotation-')) { - return true; + window.document.addEventListener("DOMContentLoaded", function (event) { + const tabsets = window.document.querySelectorAll(".panel-tabset-tabby") + tabsets.forEach(function(tabset) { + const tabby = new Tabby('#' + tabset.id); + }); + const isCodeAnnotation = (el) => { + for (const clz of el.classList) { + if (clz.startsWith('code-annotation-')) { + return true; + } } + return false; } - return false; - } - const onCopySuccess = function(e) { - // button target - const button = e.trigger; - // don't keep focus - button.blur(); - // flash "checked" - button.classList.add('code-copy-button-checked'); - var currentTitle = button.getAttribute("title"); - button.setAttribute("title", "Copied!"); - let tooltip; - if (window.bootstrap) { - button.setAttribute("data-bs-toggle", "tooltip"); - button.setAttribute("data-bs-placement", "left"); - button.setAttribute("data-bs-title", "Copied!"); - tooltip = new bootstrap.Tooltip(button, - { trigger: "manual", - customClass: "code-copy-button-tooltip", - offset: [0, -8]}); - tooltip.show(); - } - setTimeout(function() { - if (tooltip) { - tooltip.hide(); - button.removeAttribute("data-bs-title"); - button.removeAttribute("data-bs-toggle"); - button.removeAttribute("data-bs-placement"); + const onCopySuccess = function(e) { + // button target + const button = e.trigger; + // don't keep focus + button.blur(); + // flash "checked" + button.classList.add('code-copy-button-checked'); + var currentTitle = button.getAttribute("title"); + button.setAttribute("title", "Copied!"); + let tooltip; + if (window.bootstrap) { + button.setAttribute("data-bs-toggle", "tooltip"); + button.setAttribute("data-bs-placement", "left"); + button.setAttribute("data-bs-title", "Copied!"); + tooltip = new bootstrap.Tooltip(button, + { trigger: "manual", + customClass: "code-copy-button-tooltip", + offset: [0, -8]}); + tooltip.show(); } - button.setAttribute("title", currentTitle); - button.classList.remove('code-copy-button-checked'); - }, 1000); - // clear code selection - e.clearSelection(); - } - const getTextToCopy = function(trigger) { - const codeEl = trigger.previousElementSibling.cloneNode(true); - for (const childEl of codeEl.children) { - if (isCodeAnnotation(childEl)) { - childEl.remove(); + setTimeout(function() { + if (tooltip) { + tooltip.hide(); + button.removeAttribute("data-bs-title"); + button.removeAttribute("data-bs-toggle"); + button.removeAttribute("data-bs-placement"); } - } - return codeEl.innerText; - } - const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', { - text: getTextToCopy - }); - clipboard.on('success', onCopySuccess); - if (window.document.getElementById('quarto-embedded-source-code-modal')) { - const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', { - text: getTextToCopy, - container: window.document.getElementById('quarto-embedded-source-code-modal') - }); - clipboardModal.on('success', onCopySuccess); - } - var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//); - var mailtoRegex = new RegExp(/^mailto:/); - var filterRegex = new RegExp("https:\/\/workshops\.tidymodels\.org"); - var isInternal = (href) => { - return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href); + button.setAttribute("title", currentTitle); + button.classList.remove('code-copy-button-checked'); + }, 1000); + // clear code selection + e.clearSelection(); } - // Inspect non-navigation links and adorn them if external - var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)'); - for (var i=0; i { + return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href); + } + // Inspect non-navigation links and adorn them if external + var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)'); + for (var i=0; i { - const parentEl = el.parentElement; - if (parentEl) { - const cites = parentEl.dataset.cites; - if (cites) { - return { - el, - cites: cites.split(' ') - }; + const findCites = (el) => { + const parentEl = el.parentElement; + if (parentEl) { + const cites = parentEl.dataset.cites; + if (cites) { + return { + el, + cites: cites.split(' ') + }; + } else { + return findCites(el.parentElement) + } } else { - return findCites(el.parentElement) + return undefined; } - } else { - return undefined; - } - }; - var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]'); - for (var i=0; i + }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/extras-vetiver.html b/docs/archive/2025-08-nyr/slides/extras-vetiver.html index 3b9f4295..cf6afbb9 100644 --- a/docs/archive/2025-08-nyr/slides/extras-vetiver.html +++ b/docs/archive/2025-08-nyr/slides/extras-vetiver.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – Extras - Model deployment @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -566,190 +567,172 @@

Your turn

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/figures/autoplot-1.svg b/docs/archive/2025-08-nyr/slides/figures/autoplot-1.svg deleted file mode 100644 index 8a33a777..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/autoplot-1.svg +++ /dev/null @@ -1,709 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Trees - - - - - - - - - - -agent hash (log-2) - - - - - - - - - - -company hash (log-2) - - - - - - - - - - -Learning Rate (log-10) - - - - - - - - - - -Minimal Node Size - - - - - - - - - - -mae - - - - - - - - - - -rsq - - - - - - - -0 -500 -1000 -1500 -2000 - - - - - - -3 -4 -5 -6 -7 -8 - - - - - - -3 -4 -5 -6 -7 -8 - - - - --10.0 --7.5 --5.0 --2.5 - - - - -10 -20 -30 -40 -10 -20 -30 -40 -50 - - - - - -0.85 -0.90 -0.95 - - - - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/autoplot-marginals-1.svg b/docs/archive/2025-08-nyr/slides/figures/autoplot-marginals-1.svg deleted file mode 100644 index 841fbaed..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/autoplot-marginals-1.svg +++ /dev/null @@ -1,400 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Minimum Loss Reduction (log-10) - - - - - - - - - - -Tree Depth - - - - - - - - - - - - - - - - - - - -# Iterations Before Stopping - - - - - - - - - - -Learning Rate (log-10) - - - - - - - - - - -Minimal Node Size - - - - - - - --10.0 --7.5 --5.0 --2.5 -0.0 - - - -4 -8 -12 - - - - -5 -10 -15 -20 - - - - - --5 --4 --3 --2 --1 - - - - -10 -20 -30 -40 -10 -20 -30 -40 -50 - - - - - -10 -20 -30 -40 -50 - - - - - -mae - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/autoplot-param-1.svg b/docs/archive/2025-08-nyr/slides/figures/autoplot-param-1.svg deleted file mode 100644 index 1c009880..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/autoplot-param-1.svg +++ /dev/null @@ -1,622 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Minimum Loss Reduction (log-10) - - - - - - - - - - -Tree Depth - - - - - - - - - - - - - - - - - - - -# Iterations Before Stopping - - - - - - - - - - -Learning Rate (log-10) - - - - - - - - - - -Minimal Node Size - - - - - - - - - - - - - - -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 - - - - - - - - - - - - -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 - - - - - - - - - - - - -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -10 -20 -30 -40 - - - - --5 --4 --3 --2 --1 - - - - - -4 -8 -12 - - - -5 -10 -15 -20 - - - - --10.0 --7.5 --5.0 --2.5 -0.0 - - - - - -Iteration - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/autoplot-perf-1.svg b/docs/archive/2025-08-nyr/slides/figures/autoplot-perf-1.svg deleted file mode 100644 index ec21bafa..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/autoplot-perf-1.svg +++ /dev/null @@ -1,164 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10 -20 -30 -40 -50 - - - - - - - - - - - - - - - - - -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -Iteration -mae - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/autoplot-perf-zoomed-1.svg b/docs/archive/2025-08-nyr/slides/figures/autoplot-perf-zoomed-1.svg deleted file mode 100644 index 87265a56..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/autoplot-perf-zoomed-1.svg +++ /dev/null @@ -1,159 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -9 -10 -11 -12 -13 -14 - - - - - - - - - - - - - - - - - - -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -Iteration -mae - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/calibration-forested-1.svg b/docs/archive/2025-08-nyr/slides/figures/calibration-forested-1.svg deleted file mode 100644 index 3ab91552..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/calibration-forested-1.svg +++ /dev/null @@ -1,8876 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 -Bin Midpoint -Event Rate - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/conf-mat-plot-1.svg b/docs/archive/2025-08-nyr/slides/figures/conf-mat-plot-1.svg deleted file mode 100644 index 11d48b44..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/conf-mat-plot-1.svg +++ /dev/null @@ -1,64 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -5884 -438 -869 -1558 - - - -No -Yes - - - - -Yes -No -Truth -Prediction - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/cubist-autoplot-1.svg b/docs/archive/2025-08-nyr/slides/figures/cubist-autoplot-1.svg deleted file mode 100644 index 86bfe13f..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/cubist-autoplot-1.svg +++ /dev/null @@ -1,146 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Components - - - - - - - - - - -# Nearest Neighbors - - - - - - -1 -2 -3 -4 - - - - -0.0 -2.5 -5.0 -7.5 -0.8 -1.0 -1.2 - - - -mae - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/early-stopping-1.svg b/docs/archive/2025-08-nyr/slides/figures/early-stopping-1.svg deleted file mode 100644 index 661ad5c5..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/early-stopping-1.svg +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.5 -0.6 -0.7 -0.8 - - - - - - - -4 -8 -12 -iterations -ROC AUC - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/effect-compare-1.svg b/docs/archive/2025-08-nyr/slides/figures/effect-compare-1.svg deleted file mode 100644 index 1cb8fb4d..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/effect-compare-1.svg +++ /dev/null @@ -1,198 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -50 -100 -150 -200 - - - - - - - - -50 -100 -150 -200 -ADR Sample Mean -Estimated via Effects Encoding - -num_reservations - - - - - - -300 -600 -900 - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/effects-adr-1.svg b/docs/archive/2025-08-nyr/slides/figures/effects-adr-1.svg deleted file mode 100644 index 29e722c3..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/effects-adr-1.svg +++ /dev/null @@ -1,290 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/effects-again-1.svg b/docs/archive/2025-08-nyr/slides/figures/effects-again-1.svg deleted file mode 100644 index 2dd80863..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/effects-again-1.svg +++ /dev/null @@ -1,351 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/effects-again-2.svg b/docs/archive/2025-08-nyr/slides/figures/effects-again-2.svg deleted file mode 100644 index 29e722c3..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/effects-again-2.svg +++ /dev/null @@ -1,290 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/effects-freq-1.svg b/docs/archive/2025-08-nyr/slides/figures/effects-freq-1.svg deleted file mode 100644 index 2dd80863..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/effects-freq-1.svg +++ /dev/null @@ -1,351 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/forested-by-precip-annual-1.svg b/docs/archive/2025-08-nyr/slides/figures/forested-by-precip-annual-1.svg deleted file mode 100644 index 2884b573..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/forested-by-precip-annual-1.svg +++ /dev/null @@ -1,137 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -500 -1000 -1500 - - - - - - -1500 -2000 -precip_annual -count - -forested - - - - -Yes -No - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/forested-by-precip-annual-fill-1.svg b/docs/archive/2025-08-nyr/slides/figures/forested-by-precip-annual-fill-1.svg deleted file mode 100644 index 4eab2854..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/forested-by-precip-annual-fill-1.svg +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - -1500 -2000 -precip_annual -count - -forested - - - - -Yes -No - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/forested-by-tree-no-tree-1.svg b/docs/archive/2025-08-nyr/slides/figures/forested-by-tree-no-tree-1.svg deleted file mode 100644 index e45f2920..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/forested-by-tree-no-tree-1.svg +++ /dev/null @@ -1,78 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -2000 -4000 -6000 - - - - - - -Yes -No -forested -count - -tree_no_tree - - - - -Tree -No tree - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/forested-forested-by-lat-lon-1.svg b/docs/archive/2025-08-nyr/slides/figures/forested-forested-by-lat-lon-1.svg deleted file mode 100644 index 6c031b5c..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/forested-forested-by-lat-lon-1.svg +++ /dev/null @@ -1,8842 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -31 -32 -33 -34 -35 - - - - - - - - - - --85 --84 --83 --82 --81 -lon -lat - -forested - - - - -Yes -No - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/forested-forested-counts-1.svg b/docs/archive/2025-08-nyr/slides/figures/forested-forested-counts-1.svg deleted file mode 100644 index f95711c8..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/forested-forested-counts-1.svg +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -2000 -4000 -6000 - - - - - - -Yes -No -forested -count - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/gp-iter-0-1.svg b/docs/archive/2025-08-nyr/slides/figures/gp-iter-0-1.svg deleted file mode 100644 index 95e90b26..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/gp-iter-0-1.svg +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10 -20 -30 -40 -50 - - - - - - - - -1e-04 -1e-02 -1e+00 -Learning Rate -MAE - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/gp-iter-0-ei-1.svg b/docs/archive/2025-08-nyr/slides/figures/gp-iter-0-ei-1.svg deleted file mode 100644 index 34cb3c21..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/gp-iter-0-ei-1.svg +++ /dev/null @@ -1,80 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - -1e-04 -1e-02 -1e+00 -Learning Rate -Expected Improvement - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/gp-iter-1-1.svg b/docs/archive/2025-08-nyr/slides/figures/gp-iter-1-1.svg deleted file mode 100644 index 006e06ad..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/gp-iter-1-1.svg +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10 -20 -30 -40 -50 - - - - - - - - -1e-04 -1e-02 -1e+00 -Learning Rate -MAE - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/gp-iter-1-ei-1.svg b/docs/archive/2025-08-nyr/slides/figures/gp-iter-1-ei-1.svg deleted file mode 100644 index 9d804bc9..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/gp-iter-1-ei-1.svg +++ /dev/null @@ -1,81 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - -1e-04 -1e-02 -1e+00 -Learning Rate -Expected Improvement - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/grid-large-1.svg b/docs/archive/2025-08-nyr/slides/figures/grid-large-1.svg deleted file mode 100644 index f2a9ed28..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/grid-large-1.svg +++ /dev/null @@ -1,78 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10 -20 -30 -40 -50 - - - - - - - - -1e-04 -1e-02 -1e+00 -Learning Rate -MAE (resampled) - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/grid-large-sampled-1.svg b/docs/archive/2025-08-nyr/slides/figures/grid-large-sampled-1.svg deleted file mode 100644 index 75626c6b..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/grid-large-sampled-1.svg +++ /dev/null @@ -1,81 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10 -20 -30 -40 -50 - - - - - - - - -1e-04 -1e-02 -1e+00 -Learning Rate -MAE (resampled) - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/grid-types-1.svg b/docs/archive/2025-08-nyr/slides/figures/grid-types-1.svg deleted file mode 100644 index e83d6475..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/grid-types-1.svg +++ /dev/null @@ -1,354 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Regular (balanced) - - - - - - - - - - -Regular (unbalanced) - - - - - - - - - - -Random - - - - - - - - - - -SFD: Latin Hypercube - - - - - - - - - - -SFD: Uniform - - - - - - - -0 -500 -1000 -1500 -2000 - - - - - -0 -500 -1000 -1500 -2000 - - - - - -0 -500 -1000 -1500 -2000 - - - - - -0 -500 -1000 -1500 -2000 - - - - - -0 -500 -1000 -1500 -2000 -1e-08 -1e-05 -1e-02 - - - -# Trees -Learning Rate - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/lgb-cal-plot-1.svg b/docs/archive/2025-08-nyr/slides/figures/lgb-cal-plot-1.svg deleted file mode 100644 index dc49021e..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/lgb-cal-plot-1.svg +++ /dev/null @@ -1,3843 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -100 -200 -300 -400 - - - - - - - - - - -0 -100 -200 -300 -400 -Observed -Predicted - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/lm-cal-plot-1.svg b/docs/archive/2025-08-nyr/slides/figures/lm-cal-plot-1.svg deleted file mode 100644 index 8cd4c019..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/lm-cal-plot-1.svg +++ /dev/null @@ -1,3845 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -100 -200 -300 -400 - - - - - - - - - - -0 -100 -200 -300 -400 -Observed -Predicted - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/obs-pred-1.svg b/docs/archive/2025-08-nyr/slides/figures/obs-pred-1.svg deleted file mode 100644 index 0d637b03..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/obs-pred-1.svg +++ /dev/null @@ -1,101 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -5 -10 -15 -20 - - - - - - - - -5 -10 -15 -20 -ridership -.pred - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/parallel-speedup-1.svg b/docs/archive/2025-08-nyr/slides/figures/parallel-speedup-1.svg deleted file mode 100644 index 5cbf85d7..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/parallel-speedup-1.svg +++ /dev/null @@ -1,247 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -none - - - - - - - - - - -light - - - - - - - - - - -expensive - - - - - - -5 -10 -15 -20 - - - - -5 -10 -15 -20 - - - - -5 -10 -15 -20 -5 -10 -15 -20 - - - - -Number of Workers -Speed-up - -parallel_over - - - - - - -everything -resamples - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-logistic-reg-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-logistic-reg-1.svg deleted file mode 100644 index 284082c8..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-logistic-reg-1.svg +++ /dev/null @@ -1,122 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - --2 -0 -2 -A -estimate - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-logistic-reg-2-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-logistic-reg-2-1.svg deleted file mode 100644 index 284082c8..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-logistic-reg-2-1.svg +++ /dev/null @@ -1,122 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - --2 -0 -2 -A -estimate - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-logistic-reg-3-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-logistic-reg-3-1.svg deleted file mode 100644 index 729a0068..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-logistic-reg-3-1.svg +++ /dev/null @@ -1,122 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - --2 -0 -2 -A -estimate - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-race-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-race-1.svg deleted file mode 100644 index 5c11f1db..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-race-1.svg +++ /dev/null @@ -1,134 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -10 -20 -30 -40 -50 - - - - - - - - - - -2 -4 -6 -8 -10 -Analysis Stage -mae - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-rates-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-rates-1.svg deleted file mode 100644 index 50864b6e..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-rates-1.svg +++ /dev/null @@ -1,121 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - --2 -0 -2 -A -estimate - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-1.svg deleted file mode 100644 index bdf17957..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-1.svg +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - -A < 0.041 - - - -one -0.54 -100% -two -0.26 -51% -one -0.83 -49% - - -yes -no - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-2-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-2-1.svg deleted file mode 100644 index bdf17957..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-2-1.svg +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - -A < 0.041 - - - -one -0.54 -100% -two -0.26 -51% -one -0.83 -49% - - -yes -no - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-3-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-3-1.svg deleted file mode 100644 index bdf17957..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-3-1.svg +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - -A < 0.041 - - - -one -0.54 -100% -two -0.26 -51% -one -0.83 -49% - - -yes -no - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-4-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-4-1.svg deleted file mode 100644 index bc35a8c7..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-tree-fit-4-1.svg +++ /dev/null @@ -1,63 +0,0 @@ - - - - - - - - - - - - - - - - - - - -county = Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson -canopy_cover >= 42 - - - - - -Yes -0.28 -100% -Yes -0.18 -64% -Yes -0.45 -36% -Yes -0.34 -20% -No -0.59 -16% - - -yes -no - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-tree-preds-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-tree-preds-1.svg deleted file mode 100644 index 0be4e607..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-tree-preds-1.svg +++ /dev/null @@ -1,122 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - --2 -0 -2 -A -estimate - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/plot-tree-preds-2-1.svg b/docs/archive/2025-08-nyr/slides/figures/plot-tree-preds-2-1.svg deleted file mode 100644 index ffbce0dc..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/plot-tree-preds-2-1.svg +++ /dev/null @@ -1,122 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - --2 -0 -2 -A -estimate - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/roc-curve-1-1.svg b/docs/archive/2025-08-nyr/slides/figures/roc-curve-1-1.svg deleted file mode 100644 index 7750f1ca..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/roc-curve-1-1.svg +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 -1 - specificity -sensitivity - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/roc-curve-2-1.svg b/docs/archive/2025-08-nyr/slides/figures/roc-curve-2-1.svg deleted file mode 100644 index 7750f1ca..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/roc-curve-2-1.svg +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 -1 - specificity -sensitivity - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/roc-curve-code-1.svg b/docs/archive/2025-08-nyr/slides/figures/roc-curve-code-1.svg deleted file mode 100644 index 7750f1ca..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/roc-curve-code-1.svg +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 -1 - specificity -sensitivity - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/separation-forested-1.svg b/docs/archive/2025-08-nyr/slides/figures/separation-forested-1.svg deleted file mode 100644 index 0875c9f9..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/separation-forested-1.svg +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -2 -4 -6 - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 -Predicted probability of forested -density - -forested - - - - -Yes -No - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/set-results-1.svg b/docs/archive/2025-08-nyr/slides/figures/set-results-1.svg deleted file mode 100644 index a1f2e37a..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/set-results-1.svg +++ /dev/null @@ -1,232 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1.0 -1.5 -2.0 -2.5 - - - - - - - - -0 -10 -20 -30 -Workflow Rank -mae - -preprocessor - - -recipe - -model - - - - - - - - - -cubist_rules -linear_reg -mars - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/sfd-1.svg b/docs/archive/2025-08-nyr/slides/figures/sfd-1.svg deleted file mode 100644 index 2bc85d01..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/sfd-1.svg +++ /dev/null @@ -1,108 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1e-05 -1e-04 -1e-03 -1e-02 -1e-01 - - - - - - - - - - -0 -25 -50 -75 -100 -trees -learn_rate - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/stack-autoplot-1.svg b/docs/archive/2025-08-nyr/slides/figures/stack-autoplot-1.svg deleted file mode 100644 index acede092..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/stack-autoplot-1.svg +++ /dev/null @@ -1,258 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -rsq - - - - - - - - - - -rmse - - - - - - - - - - -num_members - - - - - -1e-05 -1e-03 -1e-01 -3.84 -3.87 -3.90 -3.93 -3.96 - - - - - -1.875 -1.876 -1.877 -1.878 - - - - -0.927025 -0.927030 -0.927035 -0.927040 -0.927045 - - - - - -penalty -mean - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/stack-members-1.svg b/docs/archive/2025-08-nyr/slides/figures/stack-members-1.svg deleted file mode 100644 index df3859e2..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/stack-members-1.svg +++ /dev/null @@ -1,91 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1 -2 -3 -4 -5 - - - - - - - - - -0.0 -0.1 -0.2 -0.3 -Stacking Coefficient -Member - -model - - - - -cubist_rules -linear_reg -penalty = 0.00348 - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/test-cal-1.svg b/docs/archive/2025-08-nyr/slides/figures/test-cal-1.svg deleted file mode 100644 index 9591a68d..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/test-cal-1.svg +++ /dev/null @@ -1,1339 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -100 -200 -300 -400 - - - - - - - - -100 -200 -300 -400 -Observed -Predicted - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/test-train-split-1.svg b/docs/archive/2025-08-nyr/slides/figures/test-train-split-1.svg deleted file mode 100644 index 6dced1a1..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/test-train-split-1.svg +++ /dev/null @@ -1,112 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Testing -Training -Original - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/thresholds-1.svg b/docs/archive/2025-08-nyr/slides/figures/thresholds-1.svg deleted file mode 100644 index 696412e6..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/thresholds-1.svg +++ /dev/null @@ -1,101 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 -event threshold - -statistic - - - - -sensitivity -specificity - - - diff --git a/docs/archive/2025-08-nyr/slides/figures/two-candidates-1.svg b/docs/archive/2025-08-nyr/slides/figures/two-candidates-1.svg deleted file mode 100644 index cdae2aa4..00000000 --- a/docs/archive/2025-08-nyr/slides/figures/two-candidates-1.svg +++ /dev/null @@ -1,90 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Current -best - - - -0.0 -0.2 -0.4 -0.6 - - - - - - - - -0 -5 -10 -15 -MAE -y - -Choose: - - - - -candidate 1 -candidate 2 - - - diff --git a/docs/archive/2025-08-nyr/slides/hexes/bonsai.png b/docs/archive/2025-08-nyr/slides/hexes/bonsai.png index 68eb02c0..48fe2312 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/bonsai.png and b/docs/archive/2025-08-nyr/slides/hexes/bonsai.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/broom.png b/docs/archive/2025-08-nyr/slides/hexes/broom.png index e7a825fc..3a5744db 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/broom.png and b/docs/archive/2025-08-nyr/slides/hexes/broom.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/dials.png b/docs/archive/2025-08-nyr/slides/hexes/dials.png index a951469d..1385521e 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/dials.png and b/docs/archive/2025-08-nyr/slides/hexes/dials.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/dplyr.png b/docs/archive/2025-08-nyr/slides/hexes/dplyr.png index ce908296..a545ea68 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/dplyr.png and b/docs/archive/2025-08-nyr/slides/hexes/dplyr.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/embed.png b/docs/archive/2025-08-nyr/slides/hexes/embed.png index bf1f2b81..f1fcfecb 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/embed.png and b/docs/archive/2025-08-nyr/slides/hexes/embed.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/finetune.png b/docs/archive/2025-08-nyr/slides/hexes/finetune.png index 7c869852..780ae3b6 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/finetune.png and b/docs/archive/2025-08-nyr/slides/hexes/finetune.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/ggplot2.png b/docs/archive/2025-08-nyr/slides/hexes/ggplot2.png index 6392508d..b4a7b0ea 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/ggplot2.png and b/docs/archive/2025-08-nyr/slides/hexes/ggplot2.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/parsnip.png b/docs/archive/2025-08-nyr/slides/hexes/parsnip.png index 3ec8867d..70c4495f 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/parsnip.png and b/docs/archive/2025-08-nyr/slides/hexes/parsnip.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/probably.png b/docs/archive/2025-08-nyr/slides/hexes/probably.png index c3614ff6..d665fe9b 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/probably.png and b/docs/archive/2025-08-nyr/slides/hexes/probably.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/recipes.png b/docs/archive/2025-08-nyr/slides/hexes/recipes.png index 423054a6..9dc0889a 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/recipes.png and b/docs/archive/2025-08-nyr/slides/hexes/recipes.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/rsample.png b/docs/archive/2025-08-nyr/slides/hexes/rsample.png index cf2967d4..9048167f 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/rsample.png and b/docs/archive/2025-08-nyr/slides/hexes/rsample.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/rules.png b/docs/archive/2025-08-nyr/slides/hexes/rules.png index d1bc5066..e67ab851 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/rules.png and b/docs/archive/2025-08-nyr/slides/hexes/rules.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/stacks.png b/docs/archive/2025-08-nyr/slides/hexes/stacks.png index 8b6ef7f2..b9ecd09b 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/stacks.png and b/docs/archive/2025-08-nyr/slides/hexes/stacks.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/textrecipes.png b/docs/archive/2025-08-nyr/slides/hexes/textrecipes.png index 7b16f6c1..d8062c1a 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/textrecipes.png and b/docs/archive/2025-08-nyr/slides/hexes/textrecipes.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/tidymodels.png b/docs/archive/2025-08-nyr/slides/hexes/tidymodels.png index 2d648d2b..536f8e83 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/tidymodels.png and b/docs/archive/2025-08-nyr/slides/hexes/tidymodels.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/tune.png b/docs/archive/2025-08-nyr/slides/hexes/tune.png index b0c705f4..9643fd23 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/tune.png and b/docs/archive/2025-08-nyr/slides/hexes/tune.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/vetiver.png b/docs/archive/2025-08-nyr/slides/hexes/vetiver.png index 1d74ad66..79b798a3 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/vetiver.png and b/docs/archive/2025-08-nyr/slides/hexes/vetiver.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/workflows.png b/docs/archive/2025-08-nyr/slides/hexes/workflows.png index 1d734f4f..4fe25624 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/workflows.png and b/docs/archive/2025-08-nyr/slides/hexes/workflows.png differ diff --git a/docs/archive/2025-08-nyr/slides/hexes/yardstick.png b/docs/archive/2025-08-nyr/slides/hexes/yardstick.png index 7c32cf94..e3c23b58 100644 Binary files a/docs/archive/2025-08-nyr/slides/hexes/yardstick.png and b/docs/archive/2025-08-nyr/slides/hexes/yardstick.png differ diff --git a/docs/archive/2025-08-nyr/slides/intro-01-introduction.html b/docs/archive/2025-08-nyr/slides/intro-01-introduction.html index f0e161e1..3d280c8e 100644 --- a/docs/archive/2025-08-nyr/slides/intro-01-introduction.html +++ b/docs/archive/2025-08-nyr/slides/intro-01-introduction.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 1 - Introduction @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -957,190 +958,172 @@

Our versions

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/intro-02-data-budget.html b/docs/archive/2025-08-nyr/slides/intro-02-data-budget.html index 5bc4277f..38666a59 100644 --- a/docs/archive/2025-08-nyr/slides/intro-02-data-budget.html +++ b/docs/archive/2025-08-nyr/slides/intro-02-data-budget.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 2 - Your data budget @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -831,190 +832,172 @@

The whole game - status update

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html b/docs/archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html index f788e791..68b3967b 100644 --- a/docs/archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html +++ b/docs/archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 3 - What makes a model? @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -1161,190 +1162,172 @@

The whole game - status update

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/intro-04-evaluating-models.html b/docs/archive/2025-08-nyr/slides/intro-04-evaluating-models.html index 0a0ff29e..1a8d2a32 100644 --- a/docs/archive/2025-08-nyr/slides/intro-04-evaluating-models.html +++ b/docs/archive/2025-08-nyr/slides/intro-04-evaluating-models.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 4 - Evaluating models @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -1629,190 +1630,172 @@

The whole game

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/intro-05-tuning-models.html b/docs/archive/2025-08-nyr/slides/intro-05-tuning-models.html index c80ed8b3..eecc898c 100644 --- a/docs/archive/2025-08-nyr/slides/intro-05-tuning-models.html +++ b/docs/archive/2025-08-nyr/slides/intro-05-tuning-models.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 5 - Tuning models @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -622,190 +623,172 @@

Your turn

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/intro-06-wrapping-up.html b/docs/archive/2025-08-nyr/slides/intro-06-wrapping-up.html index b2119176..70574e47 100644 --- a/docs/archive/2025-08-nyr/slides/intro-06-wrapping-up.html +++ b/docs/archive/2025-08-nyr/slides/intro-06-wrapping-up.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 6 - Wrapping up @@ -29,7 +29,7 @@ vertical-align: middle; } - + @@ -450,190 +450,172 @@

Resources to keep learning

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/intro-extra-recipes.html b/docs/archive/2025-08-nyr/slides/intro-extra-recipes.html index 2d3c0b76..d8dda912 100644 --- a/docs/archive/2025-08-nyr/slides/intro-extra-recipes.html +++ b/docs/archive/2025-08-nyr/slides/intro-extra-recipes.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – Extras - Recipes @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -861,190 +862,172 @@

More information

+ }); + \ No newline at end of file diff --git a/docs/archive/2025-08-nyr/slides/intro-extra-workflowsets.html b/docs/archive/2025-08-nyr/slides/intro-extra-workflowsets.html index 19e05de7..3385f028 100644 --- a/docs/archive/2025-08-nyr/slides/intro-extra-workflowsets.html +++ b/docs/archive/2025-08-nyr/slides/intro-extra-workflowsets.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – Extras - workflowsets @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -558,190 +559,172 @@

Your turn

+ }); + \ No newline at end of file diff --git a/docs/index.html b/docs/index.html index 6c03e1d7..52b84655 100644 --- a/docs/index.html +++ b/docs/index.html @@ -2,7 +2,7 @@ - + @@ -21,8 +21,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ +html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } -pre > code.sourceCode > span { line-height: 1.25; } +pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -33,7 +34,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } -pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } +pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -64,15 +65,16 @@ - + + - + - + + }); +

diff --git a/docs/search.json b/docs/search.json index 0482f3b2..2104d666 100644 --- a/docs/search.json +++ b/docs/search.json @@ -1,7619 +1,6975 @@ [ { - "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", - "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", - "title": "Extras - workflowsets", - "section": "How can we compare multiple model workflows at once?", - "text": "How can we compare multiple model workflows at once?" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set", - "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set", - "title": "Extras - workflowsets", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nwf_set <- workflow_set(list(forested ~ .), list(tree_spec, rf_spec))\nwf_set\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", - "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", - "title": "Extras - workflowsets", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nwf_set_fit <- wf_set %>%\n workflow_map(\"fit_resamples\", resamples = forested_folds)\nwf_set_fit\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", - "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", - "title": "Extras - workflowsets", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nwf_set_fit %>%\n rank_results()\n#> # A tibble: 6 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_fo… Prepro… accura… 0.919 0.00525 10 formula rand… 1\n#> 2 formula_rand_fo… Prepro… brier_… 0.0617 0.00338 10 formula rand… 1\n#> 3 formula_rand_fo… Prepro… roc_auc 0.972 0.00310 10 formula rand… 1\n#> 4 formula_decisio… Prepro… accura… 0.894 0.00562 10 formula deci… 2\n#> 5 formula_decisio… Prepro… brier_… 0.0817 0.00434 10 formula deci… 2\n#> 6 formula_decisio… Prepro… roc_auc 0.951 0.00378 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#your-turn", - "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#your-turn", - "title": "Extras - workflowsets", - "section": "Your turn", - "text": "Your turn\n\nWhen do you think a workflow set would be useful?\nDiscuss with your neighbors!\n\n\n\n−+\n03:00" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-06-wrapping-up.html#your-turn", - "href": "archive/2024-08-posit-conf/intro-06-wrapping-up.html#your-turn", - "title": "6 - Wrapping up", - "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-06-wrapping-up.html#resources-to-keep-learning", - "href": "archive/2024-08-posit-conf/intro-06-wrapping-up.html#resources-to-keep-learning", - "title": "6 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#looking-at-predictions", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#looking-at-predictions", - "title": "4 - Evaluating models", - "section": "Looking at predictions", - "text": "Looking at predictions\n\naugment(forested_fit, new_data = forested_train)\n#> # A tibble: 5,685 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness northness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 No 0.0114 0.989 No 2016 464 -5 -99\n#> 2 Yes 0.636 0.364 Yes 2016 166 92 37\n#> 3 No 0.0114 0.989 No 2016 644 -85 -52\n#> 4 Yes 0.977 0.0226 Yes 2014 1285 4 99\n#> 5 Yes 0.977 0.0226 Yes 2013 822 87 48\n#> 6 Yes 0.808 0.192 Yes 2017 3 6 -99\n#> 7 Yes 0.977 0.0226 Yes 2014 2041 -95 28\n#> 8 Yes 0.977 0.0226 Yes 2015 1009 -8 99\n#> 9 No 0.0114 0.989 No 2017 436 -98 19\n#> 10 No 0.0114 0.989 No 2018 775 63 76\n#> # ℹ 5,675 more rows\n#> # ℹ 14 more variables: roughness <dbl>, tree_no_tree <fct>, dew_temp <dbl>,\n#> # precip_annual <dbl>, temp_annual_mean <dbl>, temp_annual_min <dbl>,\n#> # temp_annual_max <dbl>, temp_january_min <dbl>, vapor_min <dbl>,\n#> # vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix-1", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) %>%\n conf_mat(truth = forested, estimate = .pred_class)\n#> Truth\n#> Prediction Yes No\n#> Yes 2991 176\n#> No 144 2374" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix-2", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix-2", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) %>%\n conf_mat(truth = forested, estimate = .pred_class) %>%\n autoplot(type = \"heatmap\")" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) %>%\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944\n\n\n\n\n\nThere used to be a slide here calling out the pitfalls of accuracy when classes are imbalanced." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-1", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) %>%\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.954" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-2", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-2", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) %>%\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.954\n\n\n\naugment(forested_fit, new_data = forested_train) %>%\n specificity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.931" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-3", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-3", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\nforested_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(forested_fit, new_data = forested_train) %>%\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944\n#> 2 specificity binary 0.931\n#> 3 sensitivity binary 0.954" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-4", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-4", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \nMetrics and metric sets work with grouped data frames!\n\naugment(forested_fit, new_data = forested_train) %>%\n group_by(tree_no_tree) %>%\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 2 × 4\n#> tree_no_tree .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 Tree accuracy binary 0.946\n#> 2 No tree accuracy binary 0.941" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nApply the forested_metrics metric set to augment()\noutput grouped by tree_no_tree.\nDo any metrics differ substantially between groups?\n\n\n\n−+\n05:00\n\n\n\n\nThe specificity for \"Tree\" is a good bit lower than it is for \"No tree\".\nSpecificity is the proportion of negatives that are correctly identified as negatives. “Negative” is the non-event level of the outcome, i.e. “non-forested.” So, when this index classifies the plot as having a tree, the model does not do well at correctly identifying the plot as non-forested when it is indeed non-forested." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#two-class-data", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#two-class-data", - "title": "4 - Evaluating models", - "section": "Two class data", - "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#varying-the-threshold", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#varying-the-threshold", - "title": "4 - Evaluating models", - "section": "Varying the threshold", - "text": "Varying the threshold" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves", - "title": "4 - Evaluating models", - "section": "ROC curves", - "text": "ROC curves\n\n\nFor an ROC (receiver operator characteristic) curve, we plot\n\nthe false positive rate (1 - specificity) on the x-axis\nthe true positive rate (sensitivity) on the y-axis\n\nwith sensitivity and specificity calculated at all possible thresholds.\n\n\n\n\n\n\n\n\n\n\n\n\nROC curves are insensitive to class imbalance." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves-1", - "title": "4 - Evaluating models", - "section": "ROC curves", - "text": "ROC curves\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves-2", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves-2", - "title": "4 - Evaluating models", - "section": "ROC curves ", - "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(forested_fit, new_data = forested_train) %>% \n roc_curve(truth = forested, .pred_Yes) %>%\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.235 0.885 0.972\n#> 3 0.909 0.969 0.826\n\naugment(forested_fit, new_data = forested_train) %>% \n roc_auc(truth = forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.975" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curve-plot", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curve-plot", - "title": "4 - Evaluating models", - "section": "ROC curve plot ", - "text": "ROC curve plot \n\n\naugment(forested_fit, \n new_data = forested_train) %>% \n roc_curve(truth = forested, \n .pred_Yes) %>%\n autoplot()" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-1", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#brier-score", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#brier-score", - "title": "4 - Evaluating models", - "section": "Brier score", - "text": "Brier score\nWhat if we don’t turn predicted probabilities into class predictions?\n\nThe Brier score is analogous to the mean squared error in regression models:\n\\[\nBrier_{class} = \\frac{1}{N}\\sum_{i=1}^N\\sum_{k=1}^C (y_{ik} - \\hat{p}_{ik})^2\n\\]" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#brier-score-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#brier-score-1", - "title": "4 - Evaluating models", - "section": "Brier score", - "text": "Brier score\n\naugment(forested_fit, new_data = forested_train) %>% \n brier_class(truth = forested, .pred_Yes) \n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0469\n\n\nSmaller values are better, for binary classification the “bad model threshold” is about 0.25." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#separation-vs-calibration", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#separation-vs-calibration", - "title": "4 - Evaluating models", - "section": "Separation vs calibration", - "text": "Separation vs calibration\n\n\nThe ROC captures separation.\n\n\n\n\n\n\n\n\n\n\nThe Brier score captures calibration.\n\n\n\n\n\n\n\n\n\n\n\n\nGood separation: the densities don’t overlap.\nGood calibration: the calibration line follows the diagonal.\n\nCalibration plot: We bin observations according to predicted probability. In the bin for 20%-30% predicted prob, we should see an event rate of ~25% if the model is well-calibrated." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-1", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting", - "text": "Dangers of overfitting" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-2", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-2", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-3", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-3", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\nforested_fit %>%\n augment(forested_train)\n#> # A tibble: 5,685 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness northness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 No 0.0114 0.989 No 2016 464 -5 -99\n#> 2 Yes 0.636 0.364 Yes 2016 166 92 37\n#> 3 No 0.0114 0.989 No 2016 644 -85 -52\n#> 4 Yes 0.977 0.0226 Yes 2014 1285 4 99\n#> 5 Yes 0.977 0.0226 Yes 2013 822 87 48\n#> 6 Yes 0.808 0.192 Yes 2017 3 6 -99\n#> 7 Yes 0.977 0.0226 Yes 2014 2041 -95 28\n#> 8 Yes 0.977 0.0226 Yes 2015 1009 -8 99\n#> 9 No 0.0114 0.989 No 2017 436 -98 19\n#> 10 No 0.0114 0.989 No 2018 775 63 76\n#> # ℹ 5,675 more rows\n#> # ℹ 14 more variables: roughness <dbl>, tree_no_tree <fct>, dew_temp <dbl>,\n#> # precip_annual <dbl>, temp_annual_mean <dbl>, temp_annual_min <dbl>,\n#> # temp_annual_max <dbl>, temp_january_min <dbl>, vapor_min <dbl>,\n#> # vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>\n\nWe call this “resubstitution” or “repredicting the training set”" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-4", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-4", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\nforested_fit %>%\n augment(forested_train) %>%\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944\n\nWe call this a “resubstitution estimate”" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-5", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-5", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit %>%\n augment(forested_train) %>%\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-6", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-6", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit %>%\n augment(forested_train) %>%\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944\n\n\n\nforested_fit %>%\n augment(forested_test) %>%\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.886\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-2", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-2", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse augment() and a metric function to compute a classification metric like brier_class().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-7", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-7", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit %>%\n augment(forested_train) %>%\n brier_class(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0469\n\n\n\nforested_fit %>%\n augment(forested_test) %>%\n brier_class(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0888\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-1", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-3", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-3", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-2", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-2", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(forested_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5116/569]> Fold01\n#> 2 <split [5116/569]> Fold02\n#> 3 <split [5116/569]> Fold03\n#> 4 <split [5116/569]> Fold04\n#> 5 <split [5116/569]> Fold05\n#> 6 <split [5117/568]> Fold06\n#> 7 <split [5117/568]> Fold07\n#> 8 <split [5117/568]> Fold08\n#> 9 <split [5117/568]> Fold09\n#> 10 <split [5117/568]> Fold10" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-3", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-3", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWhat is in this?\n\nforested_folds <- vfold_cv(forested_train)\nforested_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <5116/569/5685>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <5116/569/5685>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <5116/569/5685>\n\n\nTalk about a list column, storing non-atomic types in dataframe" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-4", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-4", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(forested_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [4548/1137]> Fold1\n#> 2 <split [4548/1137]> Fold2\n#> 3 <split [4548/1137]> Fold3\n#> 4 <split [4548/1137]> Fold4\n#> 5 <split [4548/1137]> Fold5" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-5", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-5", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nforested_folds <- vfold_cv(forested_train, v = 10)\nforested_folds\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5116/569]> Fold01\n#> 2 <split [5116/569]> Fold02\n#> 3 <split [5116/569]> Fold03\n#> 4 <split [5116/569]> Fold04\n#> 5 <split [5116/569]> Fold05\n#> 6 <split [5117/568]> Fold06\n#> 7 <split [5117/568]> Fold07\n#> 8 <split [5117/568]> Fold08\n#> 9 <split [5117/568]> Fold09\n#> 10 <split [5117/568]> Fold10\n\n\nSet the seed when creating resamples" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", - "title": "4 - Evaluating models", - "section": "Fit our model to the resamples", - "text": "Fit our model to the resamples\n\nforested_res <- fit_resamples(forested_wflow, forested_folds)\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [5116/569]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 2 <split [5116/569]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 3 <split [5116/569]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 4 <split [5116/569]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 5 <split [5116/569]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 6 <split [5117/568]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 7 <split [5117/568]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 8 <split [5117/568]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 9 <split [5117/568]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 10 <split [5117/568]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]>" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nforested_res %>%\n collect_metrics()\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.894 10 0.00562 Preprocessor1_Model1\n#> 2 brier_class binary 0.0817 10 0.00434 Preprocessor1_Model1\n#> 3 roc_auc binary 0.951 10 0.00378 Preprocessor1_Model1\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nWe can reliably measure performance using only the training data 🎉" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#comparing-metrics", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#comparing-metrics", - "title": "4 - Evaluating models", - "section": "Comparing metrics ", - "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\nforested_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 3 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.894 10\n#> 2 brier_class 0.0817 10\n#> 3 roc_auc 0.951 10\n\n\nThe ROC AUC previously was\n\n0.97 for the training set\n0.95 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-1", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_forested <- control_resamples(save_pred = TRUE)\nforested_res <- fit_resamples(forested_wflow, forested_folds, control = ctrl_forested)\n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [5116/569]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [5116/569]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [5116/569]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [5116/569]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [5116/569]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [5117/568]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [5117/568]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [5117/568]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [5117/568]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [5117/568]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]> <tibble>" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-2", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-2", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nforested_preds <- collect_predictions(forested_res)\nforested_preds\n#> # A tibble: 5,685 × 7\n#> .pred_class .pred_Yes .pred_No id .row forested .config \n#> <fct> <dbl> <dbl> <chr> <int> <fct> <chr> \n#> 1 Yes 0.5 0.5 Fold01 2 Yes Preprocessor1_Model1\n#> 2 Yes 0.982 0.0178 Fold01 5 Yes Preprocessor1_Model1\n#> 3 No 0.00790 0.992 Fold01 9 No Preprocessor1_Model1\n#> 4 No 0.4 0.6 Fold01 14 No Preprocessor1_Model1\n#> 5 Yes 0.870 0.130 Fold01 18 Yes Preprocessor1_Model1\n#> 6 Yes 0.982 0.0178 Fold01 59 Yes Preprocessor1_Model1\n#> 7 No 0.00790 0.992 Fold01 67 No Preprocessor1_Model1\n#> 8 Yes 0.982 0.0178 Fold01 89 Yes Preprocessor1_Model1\n#> 9 No 0.00790 0.992 Fold01 94 No Preprocessor1_Model1\n#> 10 Yes 0.982 0.0178 Fold01 111 Yes Preprocessor1_Model1\n#> # ℹ 5,675 more rows" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-3", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-3", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nforested_preds %>% \n group_by(id) %>%\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.896\n#> 2 Fold02 accuracy binary 0.859\n#> 3 Fold03 accuracy binary 0.868\n#> 4 Fold04 accuracy binary 0.921\n#> 5 Fold05 accuracy binary 0.900\n#> 6 Fold06 accuracy binary 0.891\n#> 7 Fold07 accuracy binary 0.896\n#> 8 Fold08 accuracy binary 0.903\n#> 9 Fold09 accuracy binary 0.896\n#> 10 Fold10 accuracy binary 0.905\n#> # ℹ 20 more rows" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#where-are-the-fitted-models", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#where-are-the-fitted-models", - "title": "4 - Evaluating models", - "section": "Where are the fitted models? ", - "text": "Where are the fitted models? \n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [5116/569]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [5116/569]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [5116/569]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [5116/569]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [5116/569]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [5117/568]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [5117/568]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [5117/568]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [5117/568]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [5117/568]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#bootstrapping", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#bootstrapping", - "title": "4 - Evaluating models", - "section": "Bootstrapping", - "text": "Bootstrapping" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#bootstrapping-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#bootstrapping-1", - "title": "4 - Evaluating models", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(forested_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5685/2075]> Bootstrap01\n#> 2 <split [5685/2093]> Bootstrap02\n#> 3 <split [5685/2129]> Bootstrap03\n#> 4 <split [5685/2093]> Bootstrap04\n#> 5 <split [5685/2111]> Bootstrap05\n#> 6 <split [5685/2105]> Bootstrap06\n#> 7 <split [5685/2139]> Bootstrap07\n#> 8 <split [5685/2079]> Bootstrap08\n#> 9 <split [5685/2113]> Bootstrap09\n#> 10 <split [5685/2101]> Bootstrap10\n#> # ℹ 15 more rows" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-4", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-4", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the functions)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#monte-carlo-cross-validation", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#monte-carlo-cross-validation", - "title": "4 - Evaluating models", - "section": "Monte Carlo Cross-Validation ", - "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(forested_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [4263/1422]> Resample01\n#> 2 <split [4263/1422]> Resample02\n#> 3 <split [4263/1422]> Resample03\n#> 4 <split [4263/1422]> Resample04\n#> 5 <split [4263/1422]> Resample05\n#> 6 <split [4263/1422]> Resample06\n#> 7 <split [4263/1422]> Resample07\n#> 8 <split [4263/1422]> Resample08\n#> 9 <split [4263/1422]> Resample09\n#> 10 <split [4263/1422]> Resample10" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#validation-set", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#validation-set", - "title": "4 - Evaluating models", - "section": "Validation set ", - "text": "Validation set \n\nset.seed(853)\nforested_val_split <- initial_validation_split(forested)\nvalidation_set(forested_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [4264/1421]> validation\n\n\nA validation set is just another type of resample" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#random-forest-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#random-forest-1", - "title": "4 - Evaluating models", - "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this later!), as long as there are enough trees" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model-1", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-5", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-5", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-4", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-4", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nctrl_forested <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, forested_folds, control = ctrl_forested)\ncollect_metrics(rf_res)\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.918 10 0.00585 Preprocessor1_Model1\n#> 2 brier_class binary 0.0618 10 0.00337 Preprocessor1_Model1\n#> 3 roc_auc binary 0.972 10 0.00309 Preprocessor1_Model1" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update-1", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-final-fit", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-final-fit", - "title": "4 - Evaluating models", - "section": "The final fit ", - "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# forested_split has train + test info\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [5685/1422]> train/test split <tibble> <tibble> <tibble> <workflow>" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.911 Preprocessor1_Model1\n#> 2 roc_auc binary 0.970 Preprocessor1_Model1\n#> 3 brier_class binary 0.0652 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-1", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-1", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 1,422 × 7\n#> .pred_class .pred_Yes .pred_No id .row forested .config \n#> <fct> <dbl> <dbl> <chr> <int> <fct> <chr> \n#> 1 Yes 0.822 0.178 train/test split 3 No Preprocessor1…\n#> 2 Yes 0.707 0.293 train/test split 4 Yes Preprocessor1…\n#> 3 No 0.270 0.730 train/test split 7 Yes Preprocessor1…\n#> 4 Yes 0.568 0.432 train/test split 8 Yes Preprocessor1…\n#> 5 Yes 0.554 0.446 train/test split 10 Yes Preprocessor1…\n#> 6 Yes 0.970 0.0297 train/test split 11 Yes Preprocessor1…\n#> 7 Yes 0.963 0.0367 train/test split 12 Yes Preprocessor1…\n#> 8 Yes 0.947 0.0528 train/test split 14 Yes Preprocessor1…\n#> 9 Yes 0.943 0.0573 train/test split 15 Yes Preprocessor1…\n#> 10 Yes 0.977 0.0227 train/test split 19 Yes Preprocessor1…\n#> # ℹ 1,412 more rows" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-2", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-2", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 5685 \n#> Number of independent variables: 18 \n#> Mtry: 4 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.06153207\n\n\nUse this for prediction on new data, like for deploying" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game", - "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game", - "title": "4 - Evaluating models", - "section": "The whole game", - "text": "The whole game" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington", - "title": "2 - Your data budget", - "section": "Data on forests in Washington", - "text": "Data on forests in Washington\n\n\n\nThe U.S. Forest Service maintains ML models to predict whether a plot of land is “forested.”\nThis classification is important for all sorts of research, legislation, and land management purposes.\nPlots are typically remeasured every 10 years and this dataset contains the most recent measurement per plot.\nType ?forested to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/251793/forest-mountain" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington-1", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington-1", - "title": "2 - Your data budget", - "section": "Data on forests in Washington", - "text": "Data on forests in Washington\n\n\n\nN = 7,107 plots of land, one from each of 7,107 6000-acre hexagons in WA.\nA nominal outcome, forested, with levels \"Yes\" and \"No\", measured “on-the-ground.”\n18 remotely-sensed and easily-accessible predictors:\n\nnumeric variables based on weather and topography.\nnominal variables based on classifications from other governmental orgs.\n\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/67614/forest\n\n\n\nThose nominal variables are classifications similar to “forested” but from other agencies. e.g. land_type is from the European Space Agency, and is a remotely-sensed 3-class distribution based on predictions for how the land is used." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#checklist-for-predictors", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#checklist-for-predictors", - "title": "2 - Your data budget", - "section": "Checklist for predictors", - "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?\n\n\n\nre: ethics – what issues might arise from releasing the true lat and lon? In reality, these lat and lon are slightly jittered to help ensure trust with landowners who allow surveyers to come take measurements." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington-2", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington-2", - "title": "2 - Your data budget", - "section": "Data on forests in Washington", - "text": "Data on forests in Washington\n\nlibrary(tidymodels)\nlibrary(forested)\n\nforested\n#> # A tibble: 7,107 × 19\n#> forested year elevation eastness northness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 2005 881 90 43 63 Tree 0.04\n#> 2 Yes 2005 113 -25 96 30 Tree 6.4 \n#> 3 No 2005 164 -84 53 13 Tree 6.06\n#> 4 Yes 2005 299 93 34 6 No tree 4.43\n#> 5 Yes 2005 806 47 -88 35 Tree 1.06\n#> 6 Yes 2005 736 -27 -96 53 Tree 1.35\n#> 7 Yes 2005 636 -48 87 3 No tree 1.42\n#> 8 Yes 2005 224 -65 -75 9 Tree 6.39\n#> 9 Yes 2005 52 -62 78 42 Tree 6.5 \n#> 10 Yes 2005 2240 -67 -74 99 No tree -5.63\n#> # ℹ 7,097 more rows\n#> # ℹ 11 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-1", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-1", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-2", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-2", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-initial-split", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-initial-split", - "title": "2 - Your data budget", - "section": "The initial split ", - "text": "The initial split \n\nset.seed(123)\nforested_split <- initial_split(forested)\nforested_split\n#> <Training/Testing/Total>\n#> <5330/1777/7107>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#what-is-set.seed", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#what-is-set.seed", - "title": "2 - Your data budget", - "section": "What is set.seed()?", - "text": "What is set.seed()?\nTo create that split of the data, R generates “pseudo-random” numbers: while they are made to behave like random numbers, their generation is deterministic given a “seed”.\nThis allows us to reproduce results by setting that seed.\nWhich seed you pick doesn’t matter, as long as you don’t try a bunch of seeds and pick the one that gives you the best performance." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#accessing-the-data", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#accessing-the-data", - "title": "2 - Your data budget", - "section": "Accessing the data ", - "text": "Accessing the data \n\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-training-set", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-training-set", - "title": "2 - Your data budget", - "section": "The training set", - "text": "The training set\n\nforested_train\n#> # A tibble: 5,330 × 19\n#> forested year elevation eastness northness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 No 2016 464 -5 -99 7 No tree 1.71\n#> 2 Yes 2016 166 92 37 7 Tree 6 \n#> 3 No 2016 644 -85 -52 24 No tree 0.67\n#> 4 Yes 2014 1285 4 99 79 Tree 1.91\n#> 5 Yes 2013 822 87 48 68 Tree 1.95\n#> 6 Yes 2017 3 6 -99 5 Tree 7.93\n#> 7 Yes 2014 2041 -95 28 49 Tree -4.22\n#> 8 Yes 2015 1009 -8 99 72 Tree 1.72\n#> 9 No 2017 436 -98 19 10 No tree 1.8 \n#> 10 No 2018 775 63 76 103 No tree 0.62\n#> # ℹ 5,320 more rows\n#> # ℹ 11 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-test-set", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-test-set", - "title": "2 - Your data budget", - "section": "The test set ", - "text": "The test set \n🙈\n\nThere are 1777 rows and 19 columns in the test set." - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn-1", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn-1", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n−+\n05:00" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-3", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-3", - "title": "2 - Your data budget", - "section": "Data splitting and spending ", - "text": "Data splitting and spending \n\nset.seed(123)\nforested_split <- initial_split(forested, prop = 0.8)\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)\n\nnrow(forested_train)\n#> [1] 5685\nnrow(forested_test)\n#> [1] 1422" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn-2", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn-2", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nExplore the forested_train data on your own!\n\nWhat’s the distribution of the outcome, forested?\nWhat’s the distribution of numeric variables like precip_annual?\nHow does the distribution of forested differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-1", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-1", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train %>% \n ggplot(aes(x = forested)) +\n geom_bar()" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-2", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-2", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train %>% \n ggplot(aes(x = forested, fill = tree_no_tree)) +\n geom_bar()" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-3", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-3", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train %>% \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"identity\", alpha = .7)" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-4", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-4", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train %>% \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"fill\")" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-5", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-5", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train %>% \n ggplot(aes(x = lon, y = lat, col = forested)) +\n geom_point()" - }, - { - "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-whole-game---status-update", - "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-whole-game---status-update", - "title": "2 - Your data budget", - "section": "The whole game - status update", - "text": "The whole game - status update" - }, - { - "objectID": "archive/2024-08-posit-conf/index.html", - "href": "archive/2024-08-posit-conf/index.html", + "objectID": "archive/2025-08-nyr/index.html", + "href": "archive/2025-08-nyr/index.html", "title": "Machine learning with tidymodels", "section": "", - "text": "These are the materials for workshops on tidymodels offered at posit::conf 2024. The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to tidymodels and Advanced tidymodels courses.\nIntroduction to tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nAdvanced tidymodels will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." + "text": "These are the materials for workshops on tidymodels offered at the 2025 New York Data Science & AI Conference. The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to Machine Learning in R with tidymodels and Getting More Out of Feature Engineering and Tuning for Machine Learning courses.\nIntroduction to Machine Learning in R with tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nGetting More Out of Feature Engineering and Tuning for Machine Learning will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." }, { - "objectID": "archive/2024-08-posit-conf/index.html#welcome", - "href": "archive/2024-08-posit-conf/index.html#welcome", + "objectID": "archive/2025-08-nyr/index.html#welcome", + "href": "archive/2025-08-nyr/index.html#welcome", "title": "Machine learning with tidymodels", "section": "", - "text": "These are the materials for workshops on tidymodels offered at posit::conf 2024. The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to tidymodels and Advanced tidymodels courses.\nIntroduction to tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nAdvanced tidymodels will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." + "text": "These are the materials for workshops on tidymodels offered at the 2025 New York Data Science & AI Conference. The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to Machine Learning in R with tidymodels and Getting More Out of Feature Engineering and Tuning for Machine Learning courses.\nIntroduction to Machine Learning in R with tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nGetting More Out of Feature Engineering and Tuning for Machine Learning will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." }, { - "objectID": "archive/2024-08-posit-conf/index.html#is-this-workshop-for-me", - "href": "archive/2024-08-posit-conf/index.html#is-this-workshop-for-me", + "objectID": "archive/2025-08-nyr/index.html#is-this-workshop-for-me", + "href": "archive/2025-08-nyr/index.html#is-this-workshop-for-me", "title": "Machine learning with tidymodels", "section": "Is this workshop for me? ", - "text": "Is this workshop for me? \nDepending on your background, one of Introduction to tidymodels or Advanced tidymodels might serve you better than the other.\n\nIntroduction to tidymodels\nThis workshop is for you if you:\n\nare comfortable using tidyverse packages to read data into R, transform and reshape data, and make a variety of graphs, and\nhave had some exposure to basic statistical concepts such as linear models, residuals, etc.\n\nIntermediate or expert familiarity with modeling or machine learning is not required. Interested students who have intermediate or expert familiarity with modeling or machine learning may be interested in the Advanced tidymodels workshop.\n\n\nAdvanced tidymodels\nThis workshop is for you if you:\n\nhave the prerequisite skills listed for the Introduction to tidymodels workshops,\nhave used tidymodels packages like recipes, rsample, and parsnip, and\nhave some experience with evaluating statistical models using resampling techniques like v-fold cross-validation or the bootstrap.\n\nParticipants who are new to tidymodels or machine learning will benefit from taking the Introduction to tidymodels workshop before joining this one. Participants who have completed the “Introduction to tidymodels” workshop will be well-prepared for this course." + "text": "Is this workshop for me? \nDepending on your background, one of Introduction to Machine Learning in R with tidymodels or Getting More Out of Feature Engineering and Tuning for Machine Learning might serve you better than the other.\n\nIntroduction to Machine Learning in R with tidymodels\nThis workshop is for you if you:\n\nare comfortable using tidyverse packages to read data into R, transform and reshape data, and make a variety of graphs, and\nhave had some exposure to basic statistical concepts such as linear models, residuals, etc.\n\nIntermediate or expert familiarity with modeling or machine learning is not required. Interested students who have intermediate or expert familiarity with modeling or machine learning may be interested in the Getting More Out of Feature Engineering and Tuning for Machine Learning workshop.\n\n\nGetting More Out of Feature Engineering and Tuning for Machine Learning\nThis workshop is for you if you:\n\nhave the prerequisite skills listed for the Introduction to Machine Learning in R with tidymodels workshops,\nhave used tidymodels packages like recipes, rsample, and parsnip, and\nhave some experience with evaluating statistical models using resampling techniques like v-fold cross-validation or the bootstrap.\n\nParticipants who are new to tidymodels or machine learning will benefit from taking the Introduction to Machine Learning in R with tidymodels workshop before joining this one. Participants who have completed the “Introduction to Machine Learning in R with tidymodels” workshop previously will be well-prepared for this course." }, { - "objectID": "archive/2024-08-posit-conf/index.html#preparation", - "href": "archive/2024-08-posit-conf/index.html#preparation", + "objectID": "archive/2025-08-nyr/index.html#preparation", + "href": "archive/2025-08-nyr/index.html#preparation", "title": "Machine learning with tidymodels", "section": "Preparation", - "text": "Preparation\nThe process to set up your computer for either workshop will look the same. Please join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2024.04.0), available at https://posit.co/download/rstudio-desktop/\nFor all of the slides, the following R packages can be installed from the R console:\n\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"forested\", \"lightgbm\", \"lme4\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nFor the advanced course, you should install the newest version of the dials package (version 1.3.0). To check this, you can run:\n\nrlang::check_installed(\"dials\", version = \"1.3.0\")\n\nIf you’re a Windows user and encounter an error message during installation noting a missing Rtools installation, install Rtools using the installer linked here." + "text": "Preparation\nThe process to set up your computer for either workshop will look the same. Please join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2024.04.0), available at https://posit.co/download/rstudio-desktop/\nFor all of the slides, the following R packages can be installed from the R console:\n\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\",\n \"forested\", \"lightgbm\", \"lme4\", \"pak\", \"parallelly\", \"plumber\", \n \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \n \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nIf you’re a Windows user and encounter an error message during installation noting a missing Rtools installation, install Rtools using the installer linked here." }, { - "objectID": "archive/2024-08-posit-conf/index.html#slides", - "href": "archive/2024-08-posit-conf/index.html#slides", + "objectID": "archive/2025-08-nyr/index.html#slides", + "href": "archive/2025-08-nyr/index.html#slides", "title": "Machine learning with tidymodels", "section": "Slides", - "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. They are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to tidymodels\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n05: Tuning models\n06: Wrapping up\n\n\n\nAdvanced tidymodels\n\n01: Introduction\n02: Feature engineering using recipes\n03: Tuning hyperparameters (grid search)\n04: Grid search via racing\n05: Iterative search\n06: Wrapping up\n\n\n\nExtra content (time permitting)\n\nIntro: Using workflowsets\nIntro: Using recipes\nAdvanced: Transit case study (includes stacking)\nAdvanced: Effect encoding\nAdvanced: Model deployment\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." + "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. They are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to Machine Learning in R with tidymodels\n\n\nGetting More Out of Feature Engineering and Tuning for Machine Learning\n\n\nExtra content (time permitting)" }, { - "objectID": "archive/2024-08-posit-conf/index.html#code", - "href": "archive/2024-08-posit-conf/index.html#code", + "objectID": "archive/2025-08-nyr/index.html#code", + "href": "archive/2025-08-nyr/index.html#code", "title": "Machine learning with tidymodels", "section": "Code", "text": "Code\nQuarto files for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" }, { - "objectID": "archive/2024-08-posit-conf/index.html#past-workshops", - "href": "archive/2024-08-posit-conf/index.html#past-workshops", + "objectID": "archive/2025-08-nyr/index.html#past-workshops", + "href": "archive/2025-08-nyr/index.html#past-workshops", "title": "Machine learning with tidymodels", "section": "Past workshops", - "text": "Past workshops\n\nEnglish\n\nSeptember 2023 at posit::conf()\nJuly 2023 at the New York R Conference\nAugust 2022 in Reykjavik\nJuly 2022 at rstudio::conf()\n\n\n\nSpanish\n\nMarch 2024 at conectaR" + "text": "Past workshops\n\nEnglish\n\nAugust 2024 at posit::conf()\nSeptember 2023 at posit::conf()\nJuly 2023 at the New York R Conference\nAugust 2022 in Reykjavik\nJuly 2022 at rstudio::conf()\n\n\n\nSpanish\n\nMarch 2024 at conectaR" }, { - "objectID": "archive/2024-08-posit-conf/index.html#acknowledgments", - "href": "archive/2024-08-posit-conf/index.html#acknowledgments", + "objectID": "archive/2025-08-nyr/index.html#acknowledgments", + "href": "archive/2025-08-nyr/index.html#acknowledgments", "title": "Machine learning with tidymodels", "section": "Acknowledgments", "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." }, { - "objectID": "archive/2024-08-posit-conf/index.html#reuse-and-licensing", - "href": "archive/2024-08-posit-conf/index.html#reuse-and-licensing", + "objectID": "archive/2025-08-nyr/index.html#reuse-and-licensing", + "href": "archive/2025-08-nyr/index.html#reuse-and-licensing", "title": "Machine learning with tidymodels", "section": "Reuse and licensing", "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#chicago-l-train-data", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#chicago-l-train-data", - "title": "Case Study on Transportation", - "section": "Chicago L-Train data", - "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#predictors", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#predictors", - "title": "Case Study on Transportation", - "section": "Predictors", - "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#l-train-locations", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#l-train-locations", - "title": "Case Study on Transportation", - "section": "L Train Locations", - "text": "L Train Locations" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#your-turn-explore-the-data", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#your-turn-explore-the-data", - "title": "Case Study on Transportation", - "section": "Your turn: Explore the Data", - "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n−+\n05:00" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#splitting-with-chicago-data", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#splitting-with-chicago-data", - "title": "Case Study on Transportation", - "section": "Splitting with Chicago data ", - "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#time-series-resampling", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#time-series-resampling", - "title": "Case Study on Transportation", - "section": "Time series resampling", - "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#rolling-forecast-origin-resampling", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#rolling-forecast-origin-resampling", - "title": "Case Study on Transportation", - "section": "Rolling forecast origin resampling", - "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-1", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-1", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-2", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-2", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-3", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-3", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-4", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-4", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#our-resampling-object", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#our-resampling-object", - "title": "Case Study on Transportation", - "section": "Our resampling object ", - "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#feature-engineering-with-recipes", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#feature-engineering-with-recipes", - "title": "Case Study on Transportation", - "section": "Feature engineering with recipes ", - "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe", - "title": "Case Study on Transportation", - "section": "A recipe", - "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-1", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-1", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-2", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-2", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points. The help page has a good discussion about the nuances." - }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---remove-constant-columns", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---remove-constant-columns", - "title": "Case Study on Transportation", - "section": "A recipe - remove constant columns ", - "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE) %>% \n step_zv(all_nominal_predictors())" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#looking-at-the-predictors", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#looking-at-the-predictors", + "title": "Extras - Recipes", + "section": "Looking at the predictors", + "text": "Looking at the predictors\n\nforested_train\n#> # A tibble: 5,685 × 19\n#> forested year elevation eastness northness roughness tree_no_tree dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon\n#> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 No 2016 464 -5 -99 7 No tree 1.71 282 9.76 -4.44 16.6 2.96 191 1534 4 -121.\n#> 2 Yes 2016 166 92 37 7 Tree 6 1298 10.2 0.72 14.3 6.12 60 747 33 -122.\n#> 3 No 2016 644 -85 -52 24 No tree 0.67 288 8.77 -6.32 14.6 2.98 219 1396 0 -120.\n#> 4 Yes 2014 1285 4 99 79 Tree 1.91 1621 5.61 -2.48 9.48 1.73 88 545 74 -123.\n#> 5 Yes 2013 822 87 48 68 Tree 1.95 2200 8.62 -0.68 12.9 4.35 147 861 48 -121.\n#> 6 Yes 2017 3 6 -99 5 Tree 7.93 2211 10.6 3.77 14.2 7.02 34 578 79 -124.\n#> 7 Yes 2014 2041 -95 28 49 Tree -4.22 1551 0.75 -9.47 5.17 -3.66 73 481 48 -120.\n#> 8 Yes 2015 1009 -8 99 72 Tree 1.72 2396 6.59 -2.98 11.3 1.88 92 781 76 -122.\n#> 9 No 2017 436 -98 19 10 No tree 1.8 234 9.8 -4.23 16.3 3.32 178 1527 0 -119.\n#> 10 No 2018 775 63 76 103 No tree 0.62 432 8.51 -5.5 13.7 3.32 241 1237 7 -120.\n#> # ℹ 5,675 more rows\n#> # ℹ 2 more variables: lat <dbl>, land_type <fct>" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---handle-correlations", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---handle-correlations", - "title": "Case Study on Transportation", - "section": "A recipe - handle correlations ", - "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#working-with-other-models", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#working-with-other-models", + "title": "Extras - Recipes", + "section": "Working with other models", + "text": "Working with other models\nSome models can’t handle non-numeric data\n\nLinear Regression\nK Nearest Neighbors\n\n\n\nSome models struggle if numeric predictors aren’t scaled\n\nK Nearest Neighbors\nAnything using gradient descent" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#make-some-models", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#make-some-models", - "title": "Case Study on Transportation", - "section": "Make some models ", - "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) %>% set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) %>% \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#types-of-needed-preprocessing", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#types-of-needed-preprocessing", + "title": "Extras - Recipes", + "section": "Types of needed preprocessing", + "text": "Types of needed preprocessing\n\nDo qualitative predictors require a numeric encoding?\nShould columns with a single unique value be removed?\nDoes the model struggle with missing data?\nDoes the model struggle with correlated predictors?\nShould predictors be centered and scaled?\nIs it helpful to transform predictors to be more symmetric?\n\n\nhttps://www.tmwr.org/pre-proc-table.html" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#process-them-on-the-resamples", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#process-them-on-the-resamples", - "title": "Case Study on Transportation", - "section": "Process them on the resamples", - "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set %>% \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing", + "title": "Extras - Recipes", + "section": "Two types of preprocessing", + "text": "Two types of preprocessing" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#how-do-the-results-look", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#how-do-the-results-look", - "title": "Case Study on Transportation", - "section": "How do the results look?", - "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 31 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.806 0.104 16 recipe cubis… 1\n#> 2 pca_cubist Preprocessor3_Model3 mae 0.990 0.114 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor4_Model2 mae 0.990 0.121 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model1 mae 1.00 0.127 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor3_Model2 mae 1.00 0.119 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor2_Model2 mae 1.02 0.118 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor1_Model3 mae 1.05 0.131 16 recipe cubis… 7\n#> 8 basic_cubist Preprocessor1_Model8 mae 1.07 0.115 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model7 mae 1.07 0.112 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model6 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 21 more rows" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing-1", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing-1", + "title": "Extras - Recipes", + "section": "Two types of preprocessing", + "text": "Two types of preprocessing" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#plot-the-results", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#plot-the-results", - "title": "Case Study on Transportation", - "section": "Plot the results ", - "text": "Plot the results \n\nautoplot(chi_res)" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#general-definitions", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#general-definitions", + "title": "Extras - Recipes", + "section": "General definitions", + "text": "General definitions\n\nData preprocessing is what you do to make your model successful.\nFeature engineering is what you do to the original predictors to make the model do the least work to perform great." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#pull-out-specific-results", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#pull-out-specific-results", - "title": "Case Study on Transportation", - "section": "Pull out specific results ", - "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n autoplot()" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#working-with-dates", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#working-with-dates", + "title": "Extras - Recipes", + "section": "Working with dates", + "text": "Working with dates\nDatetime variables are automatically converted to an integer if given as a raw predictor. To avoid this, it can be re-encoded as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nLeap year\nIndicators for holidays" }, - { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." + { + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-transformations", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-transformations", + "title": "Extras - Recipes", + "section": "Two types of transformations", + "text": "Two types of transformations\n\n\n\nStatic\n\nSquare root, log, inverse\nDummies for known levels\nDate time extractions\n\n\nTrained\n\nCentering & scaling\nImputation\nPCA\nAnything for unknown factor levels\n\n\n\nTrained methods need to calculate sufficient information to be applied again." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-1", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-1", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#the-recipes-package", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#the-recipes-package", + "title": "Extras - Recipes", + "section": "The recipes package", + "text": "The recipes package\n\n\nModular + extensible\nWorks well with pipes ,|> and %>%\nDeferred evaluation\nIsolates test data from training data\nCan do things formulas can’t" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-2", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-2", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-3", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-3", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-1", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-1", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())\n\n\nStart by calling recipe() to denote the data source and variables used." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-4", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-4", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-2", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-2", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())\n\n\nSpecify what actions to take by adding step_*()s." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-5", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-5", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-3", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-3", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%   step_normalize(all_numeric_predictors())\n\n\nUse {tidyselect} and recipes-specific selectors to denote affected variables." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#building-a-model-stack", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#building-a-model-stack", - "title": "Case Study on Transportation", - "section": "Building a model stack ", - "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe", + "title": "Extras - Recipes", + "section": "Using a recipe", + "text": "Using a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%   step_normalize(all_numeric_predictors())\n\n\nSave the recipe we like so that we can use it in various places, e.g., with different models." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#start-the-stack-and-add-members", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#start-the-stack-and-add-members", - "title": "Case Study on Transportation", - "section": "Start the stack and add members ", - "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() %>% \n add_candidates(chi_res)" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe-with-workflows", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe-with-workflows", + "title": "Extras - Recipes", + "section": "Using a recipe with workflows", + "text": "Using a recipe with workflows\nRecipes are typically combined with a model in a workflow() object:\n\n\nforested_wflow <- workflow() %>%\n  add_recipe(forested_rec) %>%\n  add_model(linear_reg())" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#estimate-weights-for-each-candidate", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#estimate-weights-for-each-candidate", - "title": "Case Study on Transportation", - "section": "Estimate weights for each candidate ", - "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack)\n\nchi_stack_res\n#> # A tibble: 4 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.410\n#> 2 basic_cubist_1_4 cubist_rules 0.266\n#> 3 pca_cubist_3_2 cubist_rules 0.193\n#> 4 pca_lm_4_1 linear_reg 0.171" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#recipes-are-estimated", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#recipes-are-estimated", + "title": "Extras - Recipes", + "section": "Recipes are estimated", + "text": "Recipes are estimated\nEvery preprocessing step in a recipe that involved calculations uses the training set. For example:\n\nLevels of a factor\nDetermination of zero-variance\nNormalization\nFeature extraction\n\nOnce a recipe is added to a workflow, this occurs when fit() is called." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#how-did-it-do", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#how-did-it-do", - "title": "Case Study on Transportation", - "section": "How did it do? ", - "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#debugging-a-recipe", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#debugging-a-recipe", + "title": "Extras - Recipes", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. forested_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4.\n\n\n\n\nAnother function, bake(), is analogous to predict(), and gives you the processed data back." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#what-does-it-use", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#what-does-it-use", - "title": "Case Study on Transportation", - "section": "What does it use? ", - "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#your-turn", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#your-turn", + "title": "Extras - Recipes", + "section": "Your turn", + "text": "Your turn\n\n\nTake the recipe and prep() then bake() it to see what the resulting data set looks like.\nTry removing steps to see how the result changes.\n\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#fit-the-required-candidate-models", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#fit-the-required-candidate-models", - "title": "Case Study on Transportation", - "section": "Fit the required candidate models", - "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#printing-a-recipe", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#printing-a-recipe", + "title": "Extras - Recipes", + "section": "Printing a recipe", + "text": "Printing a recipe\n\nforested_rec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Operations\n#> • Dummy variables from: all_nominal_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Log transformation on: canopy_cover\n#> • Centering and scaling for: all_numeric_predictors()" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#the-test-set-best-cubist-model", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#the-test-set-best-cubist-model", - "title": "Case Study on Transportation", - "section": "The test set: best Cubist model ", - "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n select_best()\n\ncubist_res <- \n chi_res %>% \n extract_workflow(\"pca_cubist\") %>% \n finalize_workflow(best_cubist) %>% \n last_fit(split = chi_split, metrics = metric_set(mae))" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#prepping-a-recipe", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#prepping-a-recipe", + "title": "Extras - Recipes", + "section": "Prepping a recipe", + "text": "Prepping a recipe\n\nprep(forested_rec)\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Training information\n#> Training data contained 5685 data points and no incomplete rows.\n#> \n#> ── Operations\n#> • Dummy variables from: tree_no_tree and land_type | Trained\n#> • Zero variance filter removed: <none> | Trained\n#> • Log transformation on: canopy_cover | Trained\n#> • Centering and scaling for: year and elevation, ... | Trained" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#the-test-set-stack-ensemble", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#the-test-set-stack-ensemble", - "title": "Case Study on Transportation", - "section": "The test set: stack ensemble", - "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) %>% \n bind_cols(chi_test)" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#baking-a-recipe", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#baking-a-recipe", + "title": "Extras - Recipes", + "section": "Baking a recipe", + "text": "Baking a recipe\n\nprep(forested_rec) %>%\n bake(new_data = forested_train)\n#> # A tibble: 5,685 × 20\n#> year elevation eastness northness roughness dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat forested\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> \n#> 1 0.206 -0.450 -0.0203 -1.38 -0.874 -0.169 -0.864 0.532 -0.403 0.959 -0.0702 0.755 1.21 -0.566 -0.175 -0.988 No \n#> 2 0.206 -1.07 1.38 0.563 -0.874 1.33 0.132 0.726 1.24 0.159 1.34 -1.00 -1.00 0.494 -0.856 1.10 Yes \n#> 3 0.206 -0.0762 -1.17 -0.711 -0.506 -0.533 -0.858 0.116 -1.00 0.248 -0.0613 1.13 0.820 -1.73 0.270 0.175 No \n#> 4 -0.413 1.26 0.109 1.45 0.683 -0.0987 0.448 -1.21 0.221 -1.56 -0.621 -0.625 -1.57 0.917 -1.52 0.654 Yes \n#> 5 -0.723 0.294 1.31 0.721 0.445 -0.0847 1.02 0.0529 0.795 -0.346 0.552 0.166 -0.683 0.690 -0.419 1.32 Yes \n#> 6 0.516 -1.41 0.138 -1.38 -0.917 2.01 1.03 0.894 2.21 0.127 1.75 -1.35 -1.48 0.951 -1.84 -0.338 Yes \n#> 7 -0.413 2.83 -1.32 0.435 0.0343 -2.25 0.380 -3.26 -2.01 -3.09 -3.03 -0.826 -1.75 0.690 -0.0159 1.65 Yes \n#> 8 -0.103 0.682 -0.0636 1.45 0.532 -0.165 1.21 -0.801 0.0620 -0.915 -0.554 -0.571 -0.908 0.931 -0.636 -1.33 Yes \n#> 9 0.516 -0.508 -1.36 0.306 -0.809 -0.137 -0.911 0.549 -0.336 0.856 0.0909 0.581 1.19 -1.73 0.760 -0.209 No \n#> 10 0.826 0.196 0.960 1.12 1.20 -0.550 -0.717 0.00667 -0.741 -0.0616 0.0909 1.43 0.373 -0.296 0.155 -0.204 No \n#> # ℹ 5,675 more rows\n#> # ℹ 3 more variables: tree_no_tree_No.tree <dbl>, land_type_Non.tree.vegetation <dbl>, land_type_Tree <dbl>" }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#compare-the-results", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#compare-the-results", - "title": "Case Study on Transportation", - "section": "Compare the results ", - "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.670 Preprocessor1_Model1\n\nstack_pred %>% mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.687" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\nOnce a recipe as been estimated, there are various bits of information saved in it.\n\nThe tidy() function can be used to get specific results from the recipe." }, { - "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#plot-the-test-set", - "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#plot-the-test-set", - "title": "Case Study on Transportation", - "section": "Plot the test set ", - "text": "Plot the test set \n\n\nlibrary(probably)\ncubist_res %>% \n collect_predictions() %>% \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#your-turn-1", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#your-turn-1", + "title": "Extras - Recipes", + "section": "Your turn", + "text": "Your turn\n\nTake a prepped recipe and use the tidy() function on it.\nUse the number argument to inspect different steps.\n\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#section", - "href": "archive/2024-08-posit-conf/annotations.html#section", - "title": "Annotations", - "section": "👀", - "text": "👀\nThis page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe-1", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe-1", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\n\nprep(forested_rec) %>%\n tidy()\n#> # A tibble: 4 × 6\n#> number operation type trained skip id \n#> <int> <chr> <chr> <lgl> <lgl> <chr> \n#> 1 1 step dummy TRUE FALSE dummy_jlmcG \n#> 2 2 step zv TRUE FALSE zv_mYCvS \n#> 3 3 step log TRUE FALSE log_eme6b \n#> 4 4 step normalize TRUE FALSE normalize_ScVef" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#data-splitting-and-spending", - "href": "archive/2024-08-posit-conf/annotations.html#data-splitting-and-spending", - "title": "Annotations", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nMore about the initial data split can be found in Chapter 3 of Applied Machine Learning for Tabular Data (AML4TD).\nIn particular, a three-way split into training, validation, and testing set can be done via\n\nset.seed(123)\ninitial_validation_split(forested, prop = c(0.6, 0.2))\n#> <Training/Validation/Testing/Total>\n#> <4264/1421/1422/7107>" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe-2", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe-2", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\n\nprep(forested_rec) %>%\n tidy(number = 1)\n#> # A tibble: 3 × 3\n#> terms columns id \n#> <chr> <chr> <chr> \n#> 1 tree_no_tree No tree dummy_jlmcG\n#> 2 land_type Non-tree vegetation dummy_jlmcG\n#> 3 land_type Tree dummy_jlmcG" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#what-is-set.seed", - "href": "archive/2024-08-posit-conf/annotations.html#what-is-set.seed", - "title": "Annotations", - "section": "What is set.seed()?", - "text": "What is set.seed()?\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe-in-tidymodels", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe-in-tidymodels", + "title": "Extras - Recipes", + "section": "Using a recipe in tidymodels", + "text": "Using a recipe in tidymodels\nThe recommended way to use a recipe in tidymodels is to use it as part of a workflow().\n\nforested_wflow <- workflow() %>% \n add_recipe(forested_rec) %>% \n add_model(linear_reg())\n\nWhen used in this way, you don’t need to worry about prep() and bake() as it is handled for you." }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#what-is-wrong-with-this", - "href": "archive/2024-08-posit-conf/annotations.html#what-is-wrong-with-this", - "title": "Annotations", - "section": "What is wrong with this?", - "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" + "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#more-information", + "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#more-information", + "title": "Extras - Recipes", + "section": "More information", + "text": "More information\n\nhttps://recipes.tidymodels.org/\nhttps://www.tmwr.org/recipes.html" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#brier-score", - "href": "archive/2024-08-posit-conf/annotations.html#brier-score", - "title": "Annotations", - "section": "Brier score", - "text": "Brier score\nThe Brier score measures how close a model probability estimate is to its best possible value (i.e., zero or one).\nIn the best case, the model is perfect, and every prediction equals 0.0 or 1.0 (depending on the true class). In this case, the Brier score is zero.\nWhen the model is uninformative and there are two classes, the worst-case values range from 0.25 to about 0.50. Imagine that the model predicts the same noninformative prediction of 50% (basically “¯\\(ツ)/¯”). In that case, every prediction is either \\((0.00 - 0.50)^2\\) or \\((1.00 - 0.50)^2\\). The average of those is 0.25.\nThere are many different ways a model can be bad though, and some of these will produce Brier scores between 0.25 and 0.50." + "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#tuning-parameters", + "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#tuning-parameters", + "title": "5 - Tuning models", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#dangers-of-overfitting", - "href": "archive/2024-08-posit-conf/annotations.html#dangers-of-overfitting", - "title": "Annotations", - "section": "Dangers of overfitting", - "text": "Dangers of overfitting\nSee the “Overfitting” chapter of AML4TD for more information." + "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters", + "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters", + "title": "5 - Tuning models", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#where-are-the-fitted-models", - "href": "archive/2024-08-posit-conf/annotations.html#where-are-the-fitted-models", - "title": "Annotations", - "section": "Where are the fitted models?", - "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" + "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters-1", + "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters-1", + "title": "5 - Tuning models", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#resampling-strategy", - "href": "archive/2024-08-posit-conf/annotations.html#resampling-strategy", - "title": "Annotations", - "section": "Resampling Strategy", - "text": "Resampling Strategy\nThese data have a time component, and while not a typical time series data set, we have the option to use a time series resampling method.\nAn example is shown in the extra slides “Case Study on Transportation”.\nConsider the agent data. Cross-validation may not group all of an agent’s data into the analysis or assessment sets. In this case, our analysis data might have future data that is later than the agent’s data in the assessment set." + "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#specifying-tuning-parameters", + "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#specifying-tuning-parameters", + "title": "5 - Tuning models", + "section": "Specifying tuning parameters", + "text": "Specifying tuning parameters\nLet’s take our previous random forest workflow and tag for tuning the minimum number of data points in each node:\n\nrf_spec <- rand_forest(min_n = tune()) %>% \n set_mode(\"classification\")\n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#different-types-of-grids", - "href": "archive/2024-08-posit-conf/annotations.html#different-types-of-grids", - "title": "Annotations", - "section": "Different types of grids", - "text": "Different types of grids\nMore on space-filling designs in Chapters 4 and 5 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences.\nIn the next version of tune (version 1.3.0) an improved set of space-filling designs will be the first choice if you ask for an automated grid." + "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#try-out-multiple-values", + "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#try-out-multiple-values", + "title": "5 - Tuning models", + "section": "Try out multiple values", + "text": "Try out multiple values\ntune_grid() works similar to fit_resamples() but covers multiple parameter values:\n\nset.seed(22)\nrf_res <- tune_grid(\n rf_wflow,\n forested_folds,\n grid = 5\n)" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#update-parameter-ranges", - "href": "archive/2024-08-posit-conf/annotations.html#update-parameter-ranges", - "title": "Annotations", - "section": "Update parameter ranges", - "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_spline_natural(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" + "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#compare-results", + "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#compare-results", + "title": "5 - Tuning models", + "section": "Compare results", + "text": "Compare results\nInspecting results and selecting the best-performing hyperparameter(s):\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 21 roc_auc binary 0.972 10 0.00295 Preprocessor1_Model4\n#> 2 6 roc_auc binary 0.972 10 0.00303 Preprocessor1_Model2\n#> 3 31 roc_auc binary 0.972 10 0.00317 Preprocessor1_Model3\n#> 4 13 roc_auc binary 0.972 10 0.00311 Preprocessor1_Model5\n#> 5 33 roc_auc binary 0.972 10 0.00322 Preprocessor1_Model1\n\nbest_parameter <- select_best(rf_res)\nbest_parameter\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 21 Preprocessor1_Model4\n\ncollect_metrics() and autoplot() are also available." }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#early-stopping-for-boosted-trees", - "href": "archive/2024-08-posit-conf/annotations.html#early-stopping-for-boosted-trees", - "title": "Annotations", - "section": "Early stopping for boosted trees", - "text": "Early stopping for boosted trees\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the lightgbm package.\nbonsai has its own wrapper around (lightgbm::lgb.train()) called bonsai::train_lightgbm(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the lightgbm entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for lightgbm)." + "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#the-final-fit", + "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#the-final-fit", + "title": "5 - Tuning models", + "section": "The final fit", + "text": "The final fit\n\nrf_wflow <- finalize_workflow(rf_wflow, best_parameter)\n\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.906 Preprocessor1_Model1\n#> 2 roc_auc binary 0.970 Preprocessor1_Model1\n#> 3 brier_class binary 0.0656 Preprocessor1_Model1" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#gaussian-processes-and-optimization", - "href": "archive/2024-08-posit-conf/annotations.html#gaussian-processes-and-optimization", - "title": "Annotations", - "section": "Gaussian Processes and Optimization", - "text": "Gaussian Processes and Optimization\nSome other references for GP’s:\n\nChapter 5 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences\nBayesian Optimization, Chapter 3 (pdf)\nGaussian Processes for Machine Learning (pdf)" + "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#your-turn", + "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#your-turn", + "title": "5 - Tuning models", + "section": "Your turn", + "text": "Your turn\n\nModify your model workflow to tune one or more parameters.\nUse grid search to find the best parameter(s).\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#acquisition-functions", - "href": "archive/2024-08-posit-conf/annotations.html#acquisition-functions", - "title": "Annotations", - "section": "Acquisition Functions", - "text": "Acquisition Functions\nMore references:\n\nChapter 7 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences\nBayesian Optimization, Chapter 6 (pdf)\nGaussian Processes for Machine Learning" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets\nbrulee for regression using torch" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#per-agent-statistics", - "href": "archive/2024-08-posit-conf/annotations.html#per-agent-statistics", - "title": "Annotations", - "section": "Per-agent statistics", - "text": "Per-agent statistics\nThe effect encoding method essentially takes the effect of a variable, like agent, and makes a data column for that effect. In our example, affect of the agent on the ADR is quantified by a model and then added as a data column to be used in the model.\nSuppose agent Max has a single reservation in the data and it had an ADR of €200. If we used a naive estimate for Max’s effect, the model is being told that Max should always produce an effect of €200. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned agent Davis, who has taken 250 reservations with an average ADR of €100. Davis’s mean is more predictive because it is estimated with better data (i.e., more total reservations). Partial pooling leverages the entire data set and can borrow strength from all of the agents. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a agent’s data is of good quality, the partial pooling effect estimate is closer to the raw mean. Max’s data is not great and is “shrunk” towards the center of the overall average. Since there is so little known about Max’s reservation history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nSince this example has a numeric outcome, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#agent-effects", - "href": "archive/2024-08-posit-conf/annotations.html#agent-effects", - "title": "Annotations", - "section": "Agent effects", - "text": "Agent effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage though." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-1", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-1", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" }, { - "objectID": "archive/2024-08-posit-conf/annotations.html#a-recipe---handle-correlations", - "href": "archive/2024-08-posit-conf/annotations.html#a-recipe---handle-correlations", - "title": "Annotations", - "section": "A recipe - handle correlations", - "text": "A recipe - handle correlations\nIn this code chunk, What’s the story with !!stations?\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nstations is a vector of names of 20 columns that we want to use in the steps. If the list were shorter, we could type them in (e.g., c(\"col1\", \"col2\") etc.).\nThe vector lives in our global workspace, and if we are in parallel, the worker processes might not have access to stations. The !! (frequently said as “bang bang”) inserts the actual contents of the vector into the all_of() calls so that it looks like you just typed it in.\nThis means that the parallel process workers have a copy of the data in their reach and the code will run without error." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-2", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-2", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#previously---setup", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#previously---setup", - "title": "5 - Iterative Search", - "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-3", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-3", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#previously---data-usage", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#previously---data-usage", - "title": "5 - Iterative Search", - "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-4", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-4", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#our-boosting-model", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#our-boosting-model", - "title": "5 - Iterative Search", - "section": "Our Boosting Model", - "text": "Our Boosting Model\nWe used feature hashing to generate a smaller set of indicator columns to deal with the large number of levels for the agent and country predictors.\n\nTree-based models (and a few others) don’t require indicators for categorical predictors. They can split on these variables as-is.\n\nWe’ll keep all categorical predictors as factors and focus on optimizing additional boosting parameters." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-5", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-5", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#our-boosting-model-1", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#our-boosting-model-1", - "title": "5 - Iterative Search", - "section": "Our Boosting Model", - "text": "Our Boosting Model\n\nlgbm_spec <- \n boost_tree(trees = 1000, learn_rate = tune(), min_n = tune(), \n tree_depth = tune(), loss_reduction = tune(), \n stop_iter = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(avg_price_per_room ~ ., lgbm_spec)\n\nlgbm_param <- \n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(learn_rate = learn_rate(c(-5, -1)))" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-6", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-6", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#iterative-search", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#iterative-search", - "title": "5 - Iterative Search", - "section": "Iterative Search", - "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our boosted tree.\n\nWe could do something like:\nmae_pred <- lm(mae ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-7", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-7", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#iterative-search-1", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#iterative-search-1", - "title": "5 - Iterative Search", - "section": "Iterative Search", - "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between MAE and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-8", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-8", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#a-large-grid", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#a-large-grid", - "title": "5 - Iterative Search", - "section": "A Large Grid", - "text": "A Large Grid" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-1", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-1", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a logistic regression model.\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/\n\n\nExtension/Challenge: Edit this code to use a different model. For example, try using a conditional inference tree as implemented in the partykit package by changing the engine - or try an entirely different model type!\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#a-three-point-grid", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#a-three-point-grid", - "title": "5 - Iterative Search", - "section": "A Three Point Grid", - "text": "A Three Point Grid" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#models-well-be-using-today", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#models-well-be-using-today", + "title": "3 - What makes a model?", + "section": "Models we’ll be using today", + "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#gaussian-processes-and-optimization", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#gaussian-processes-and-optimization", - "title": "5 - Iterative Search", - "section": "Gaussian Processes and Optimization", - "text": "Gaussian Processes and Optimization\nWe can make a “meta-model” with a small set of historical performance results.\nGaussian Processes (GP) models are a good choice to model performance.\n\nIt is a Bayesian model so we are using Bayesian Optimization (BO).\nFor regression, we can assume that our data are multivariate normal.\nWe also define a covariance function for the variance relationship between data points. A common one is:\n\n\\[\\operatorname{cov}(\\boldsymbol{x}_i, \\boldsymbol{x}_j) = \\exp\\left(-\\frac{1}{2}|\\boldsymbol{x}_i - \\boldsymbol{x}_j|^2\\right) + \\sigma^2_{ij}\\]\n\nGPs are good because\n\nthey are flexible regression models (in the sense that splines are flexible).\nwe need to get mean and variance predictions (and they are Bayesian)\ntheir variability is based on spatial distances.\n\nSome people use random forests (with conformal variance estimates) or other methods but GPs are most popular." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#predicting-candidates", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#predicting-candidates", - "title": "5 - Iterative Search", - "section": "Predicting Candidates", - "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. MAE)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-1", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-1", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#your-turn", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#your-turn", - "title": "5 - Iterative Search", - "section": "Your turn", - "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize MAE.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-2", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-2", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nFind a sigmoid line that separates the two classes" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", - "title": "5 - Iterative Search", - "section": "GP Fit (ribbon is mean +/- 1SD)", - "text": "GP Fit (ribbon is mean +/- 1SD)" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#choosing-new-candidates", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#choosing-new-candidates", - "title": "5 - Iterative Search", - "section": "Choosing New Candidates", - "text": "Choosing New Candidates\nThis isn’t a very good fit but we can still use it.\nHow can we use the outputs to choose the next point to measure?\n\nAcquisition functions take the predicted mean and variance and use them to balance:\n\nexploration: new candidates should explore new areas.\nexploitation: new candidates must stay near existing values.\n\nExploration focuses on the variance, exploitation is about the mean." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees-1", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees-1", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#acquisition-functions", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#acquisition-functions", - "title": "5 - Iterative Search", - "section": "Acquisition Functions", - "text": "Acquisition Functions\nWe’ll use an acquisition function to select a new candidate.\nThe most popular method appears to be expected improvement (EI) above the current best results.\n\nZero at existing data points.\nThe expected improvement is integrated over all possible improvement (“expected” in the probability sense).\n\nWe would probably pick the point with the largest EI as the next point.\n(There are other functions beyond EI.)" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees-2", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees-2", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement", - "title": "5 - Iterative Search", - "section": "Expected Improvement", - "text": "Expected Improvement" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "title": "3 - What makes a model?", + "section": "All models are wrong, but some are useful!", + "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#iteration", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#iteration", - "title": "5 - Iterative Search", - "section": "Iteration", - "text": "Iteration\nOnce we pick the candidate point, we measure performance for it (e.g. resampling).\n\nAnother GP is fit, EI is recomputed, and so on.\n\nWe stop when we have completed the allowed number of iterations or if we don’t see any improvement after a pre-set number of attempts." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "title": "3 - What makes a model?", + "section": "Workflows bind preprocessors and models", + "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-fit-with-four-points", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-fit-with-four-points", - "title": "5 - Iterative Search", - "section": "GP Fit with four points", - "text": "GP Fit with four points" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#what-is-wrong-with-this", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#what-is-wrong-with-this", + "title": "3 - What makes a model?", + "section": "What is wrong with this?", + "text": "What is wrong with this?" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement-1", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement-1", - "title": "5 - Iterative Search", - "section": "Expected Improvement", - "text": "Expected Improvement" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#why-a-workflow", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#why-a-workflow", + "title": "3 - What makes a model?", + "section": "Why a workflow()? ", + "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering in Advanced tidymodels!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-evolution", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-evolution", - "title": "5 - Iterative Search", - "section": "GP Evolution", - "text": "GP Evolution" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-1", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-1", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\ntree_spec %>% \n fit(forested ~ ., data = forested_train) \n#> parsnip model object\n#> \n#> n= 5685 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 5685 2550 Yes (0.55145119 0.44854881) \n#> 2) land_type=Tree 3064 300 Yes (0.90208877 0.09791123) *\n#> 3) land_type=Barren,Non-tree vegetation 2621 371 No (0.14154903 0.85845097) \n#> 6) temp_annual_max< 13.395 347 153 Yes (0.55907781 0.44092219) \n#> 12) tree_no_tree=Tree 92 6 Yes (0.93478261 0.06521739) *\n#> 13) tree_no_tree=No tree 255 108 No (0.42352941 0.57647059) *\n#> 7) temp_annual_max>=13.395 2274 177 No (0.07783641 0.92216359) *" + }, + { + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-2", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-2", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\nworkflow() %>%\n add_formula(forested ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 5685 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 5685 2550 Yes (0.55145119 0.44854881) \n#> 2) land_type=Tree 3064 300 Yes (0.90208877 0.09791123) *\n#> 3) land_type=Barren,Non-tree vegetation 2621 371 No (0.14154903 0.85845097) \n#> 6) temp_annual_max< 13.395 347 153 Yes (0.55907781 0.44092219) \n#> 12) tree_no_tree=Tree 92 6 Yes (0.93478261 0.06521739) *\n#> 13) tree_no_tree=No tree 255 108 No (0.42352941 0.57647059) *\n#> 7) temp_annual_max>=13.395 2274 177 No (0.07783641 0.92216359) *" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement-evolution", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement-evolution", - "title": "5 - Iterative Search", - "section": "Expected Improvement Evolution", - "text": "Expected Improvement Evolution" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-3", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-3", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\nworkflow(forested ~ ., tree_spec) %>% \n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 5685 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 5685 2550 Yes (0.55145119 0.44854881) \n#> 2) land_type=Tree 3064 300 Yes (0.90208877 0.09791123) *\n#> 3) land_type=Barren,Non-tree vegetation 2621 371 No (0.14154903 0.85845097) \n#> 6) temp_annual_max< 13.395 347 153 Yes (0.55907781 0.44092219) \n#> 12) tree_no_tree=Tree 92 6 Yes (0.93478261 0.06521739) *\n#> 13) tree_no_tree=No tree 255 108 No (0.42352941 0.57647059) *\n#> 7) temp_annual_max>=13.395 2274 177 No (0.07783641 0.92216359) *" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#bo-in-tidymodels", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#bo-in-tidymodels", - "title": "5 - Iterative Search", - "section": "BO in tidymodels", - "text": "BO in tidymodels\nWe’ll use a function called tune_bayes() that has very similar syntax to tune_grid().\n\nIt has an additional initial argument for the initial set of performance estimates and parameter combinations for the GP model." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-2", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-2", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code to make a workflow with your own model of choice.\n\nExtension/Challenge: Other than formulas, what kinds of preprocessors are supported?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#initial-grid-points", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#initial-grid-points", - "title": "5 - Iterative Search", - "section": "Initial grid points", - "text": "Initial grid points\ninitial can be the results of another tune_*() function or an integer (in which case tune_grid() is used under to hood to make such an initial set of results).\n\nWe’ll run the optimization more than once, so let’s make an initial grid of results to serve as the substrate for the BO.\nI suggest at least the number of tuning parameters plus two as the initial grid for BO." + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#predict-with-your-model", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#predict-with-your-model", + "title": "3 - What makes a model?", + "section": "Predict with your model ", + "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(forested ~ ., tree_spec) %>% \n fit(data = forested_train)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#an-initial-grid", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#an-initial-grid", - "title": "5 - Iterative Search", - "section": "An Initial Grid", - "text": "An Initial Grid\n\nreg_metrics <- metric_set(mae, rsq)\n\nset.seed(12)\ninit_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = nrow(lgbm_param) + 2,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(init_res, metric = \"mae\") %>% select(-.metric, -.estimator)\n#> # A tibble: 5 × 9\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config \n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> \n#> 1 16 12 0.0136 1.91e- 3 9 10.1 10 0.196 Preprocessor1_Model4\n#> 2 9 4 0.0415 5.21e- 9 13 10.2 10 0.167 Preprocessor1_Model1\n#> 3 25 8 0.00256 9.58e-10 7 14.1 10 0.278 Preprocessor1_Model7\n#> 4 22 9 0.00154 5.77e- 6 5 19.3 10 0.326 Preprocessor1_Model5\n#> 5 32 3 0.000144 3.02e+ 1 18 47.6 10 0.387 Preprocessor1_Model6" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-3", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-3", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = forested_test)\nWhat do you notice about the structure of the result?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#bo-using-tidymodels", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#bo-using-tidymodels", - "title": "5 - Iterative Search", - "section": "BO using tidymodels", - "text": "BO using tidymodels\n\nctrl_bo <- control_bayes(verbose_iter = TRUE) # <- for demonstration\n\nset.seed(15)\nlgbm_bayes_res <-\n lgbm_wflow %>%\n tune_bayes(\n resamples = hotel_rs,\n initial = init_res, # <- initial results\n iter = 20,\n param_info = lgbm_param,\n control = ctrl_bo,\n metrics = reg_metrics\n )\n#> Optimizing mae using the expected improvement\n#> \n#> ── Iteration 1 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=10.13 (@iter 0)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=32, tree_depth=12, learn_rate=0.0178, loss_reduction=1.03e-10, stop_iter=12\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=10.08 (+/-0.175)\n#> \n#> ── Iteration 2 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=10.08 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=15, tree_depth=14, learn_rate=0.0977, loss_reduction=0.00535, stop_iter=4\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.719 (+/-0.187)\n#> \n#> ── Iteration 3 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=38, tree_depth=1, learn_rate=0.1, loss_reduction=0.0809, stop_iter=10\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=15.45 (+/-0.253)\n#> \n#> ── Iteration 4 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=32, tree_depth=1, learn_rate=0.00833, loss_reduction=1.31e-06, stop_iter=8\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=19.44 (+/-0.33)\n#> \n#> ── Iteration 5 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=18, tree_depth=8, learn_rate=0.0495, loss_reduction=1.4e-06, stop_iter=5\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.757 (+/-0.146)\n#> \n#> ── Iteration 6 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=3, tree_depth=14, learn_rate=0.0319, loss_reduction=4.02e-09, stop_iter=17\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.76 (+/-0.163)\n#> \n#> ── Iteration 7 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=6, tree_depth=8, learn_rate=0.0883, loss_reduction=1.94e-08, stop_iter=4\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.712 (+/-0.17)\n#> \n#> ── Iteration 8 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.712 (@iter 7)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=6, tree_depth=8, learn_rate=0.025, loss_reduction=7.82e-05, stop_iter=19\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.838 (+/-0.17)\n#> \n#> ── Iteration 9 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.712 (@iter 7)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=32, tree_depth=6, learn_rate=0.0737, loss_reduction=2.15e-07, stop_iter=8\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.06 (+/-0.2)\n#> \n#> ── Iteration 10 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.712 (@iter 7)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=5, tree_depth=11, learn_rate=0.0451, loss_reduction=3.45e-10, stop_iter=7\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.637 (+/-0.156)\n#> \n#> ── Iteration 11 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=7, learn_rate=0.0372, loss_reduction=2.44e-09, stop_iter=11\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.761 (+/-0.171)\n#> \n#> ── Iteration 12 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=26, tree_depth=15, learn_rate=0.00626, loss_reduction=0.00554, stop_iter=16\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.79 (+/-0.198)\n#> \n#> ── Iteration 13 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=29, tree_depth=10, learn_rate=0.0996, loss_reduction=4.5e-05, stop_iter=16\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.838 (+/-0.169)\n#> \n#> ── Iteration 14 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=12, tree_depth=13, learn_rate=0.085, loss_reduction=2.16, stop_iter=9\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.795 (+/-0.16)\n#> \n#> ── Iteration 15 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=4, tree_depth=9, learn_rate=0.0418, loss_reduction=0.00293, stop_iter=7\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.75 (+/-0.168)\n#> \n#> ── Iteration 16 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=6, tree_depth=15, learn_rate=0.0703, loss_reduction=5.15e-10, stop_iter=13\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.672 (+/-0.134)\n#> \n#> ── Iteration 17 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=27, tree_depth=15, learn_rate=0.0956, loss_reduction=3.74e-10, stop_iter=17\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.861 (+/-0.197)\n#> \n#> ── Iteration 18 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=11, learn_rate=0.0871, loss_reduction=0.00196, stop_iter=18\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.601 (+/-0.147)\n#> \n#> ── Iteration 19 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.601 (@iter 18)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=12, learn_rate=0.0991, loss_reduction=8.45e-06, stop_iter=14\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.61 (+/-0.17)\n#> \n#> ── Iteration 20 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.601 (@iter 18)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=4, tree_depth=15, learn_rate=0.0206, loss_reduction=1.46e-06, stop_iter=15\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.881 (+/-0.177)" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-4", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-4", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = forested_test)\nHow does the output compare to the output from predict()?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#best-results", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#best-results", - "title": "5 - Iterative Search", - "section": "Best results", - "text": "Best results\n\nshow_best(lgbm_bayes_res, metric = \"mae\") %>% select(-.metric, -.estimator)\n#> # A tibble: 5 × 10\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config .iter\n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> <int>\n#> 1 2 11 0.0871 1.96e- 3 18 9.60 10 0.147 Iter18 18\n#> 2 2 12 0.0991 8.45e- 6 14 9.61 10 0.170 Iter19 19\n#> 3 5 11 0.0451 3.45e-10 7 9.64 10 0.156 Iter10 10\n#> 4 6 15 0.0703 5.15e-10 13 9.67 10 0.134 Iter16 16\n#> 5 6 8 0.0883 1.94e- 8 4 9.71 10 0.170 Iter7 7" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results", - "title": "5 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\")" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-1", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-1", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\n⚠️ Never predict() with any extracted components!\n\nroundint = FALSE is only to quiet a warning" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results-1", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results-1", - "title": "5 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"parameters\")" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-2", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-2", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results-2", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results-2", - "title": "5 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\")" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-5", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-5", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\n\nExtract the model engine object from your fitted workflow and check it out.\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#enhance", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#enhance", - "title": "5 - Iterative Search", - "section": "ENHANCE", - "text": "ENHANCE\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\") +\n ylim(c(9, 14))" + "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#the-whole-game---status-update", + "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#the-whole-game---status-update", + "title": "3 - What makes a model?", + "section": "The whole game - status update", + "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#your-turn-1", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#your-turn-1", - "title": "5 - Iterative Search", - "section": "Your turn", - "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\nBonus points: Before the optimization is done, press <esc> and see what happens.\n\n\n\n−+\n10:00" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#venue-information", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#venue-information", + "title": "1 - Introduction", + "section": "Venue information", + "text": "Venue information\n\nThere are gender neutral bathrooms located on levels 3, 4, 5, 6 & 7\nA meditation/prayer room is located in 503\n(Mon & Tue 7am - 7pm, and Wed 7am - 5pm)\nA lactation room is located in 509\n(Mon & Tue 7am - 7pm, and Wed 7am - 5pm)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#notes", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#notes", - "title": "5 - Iterative Search", - "section": "Notes", - "text": "Notes\n\nStopping tune_bayes() will return the current results.\nParallel processing can still be used to more efficiently measure each candidate point.\nThere are a lot of other iterative methods that you can use.\nThe finetune package also has functions for simulated annealing search." + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#workshop-policies", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#workshop-policies", + "title": "1 - Introduction", + "section": "Workshop policies", + "text": "Workshop policies\n\nPlease review the posit::conf code of conduct, which applies to all workshops: https://posit.co/code-of-conduct\nCoC site has info on how to report a problem (in person, email, phone)\nPlease do not photograph people wearing red lanyards" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#finalizing-the-model", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#finalizing-the-model", - "title": "5 - Iterative Search", - "section": "Finalizing the Model", - "text": "Finalizing the Model\nLet’s say that we’ve tried a lot of different models and we like our lightgbm model the most.\nWhat do we do now?\n\nFinalize the workflow by choosing the values for the tuning parameters.\nFit the model on the entire training set.\nVerify performance using the test set.\nDocument and publish the model(?)" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#who-are-you", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have some exposure to basic statistical concepts like linear models and residuals\nYou do not need intermediate or expert familiarity with modeling or ML" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#locking-down-the-tuning-parameters", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#locking-down-the-tuning-parameters", - "title": "5 - Iterative Search", - "section": "Locking Down the Tuning Parameters", - "text": "Locking Down the Tuning Parameters\nWe can take the results of the Bayesian optimization and accept the best results:\n\nbest_param <- select_best(lgbm_bayes_res, metric = \"mae\")\nfinal_wflow <- \n lgbm_wflow %>% \n finalize_workflow(best_param)\nfinal_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: boost_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> avg_price_per_room ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Boosted Tree Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> min_n = 2\n#> tree_depth = 11\n#> learn_rate = 0.0871075826616985\n#> loss_reduction = 0.00195652467829182\n#> stop_iter = 18\n#> \n#> Engine-Specific Arguments:\n#> num_threads = 1\n#> \n#> Computational engine: lightgbm" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#who-are-tidymodels", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\n+ our TA today, Sara Altman!\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#the-final-fit", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#the-final-fit", - "title": "5 - Iterative Search", - "section": "The Final Fit", - "text": "The Final Fit\nWe can use individual functions:\nfinal_fit <- final_wflow %>% fit(data = hotel_train)\n\n# then predict() or augment() \n# then compute metrics\n\nRemember that there is also a convenience function to do all of this:\n\nset.seed(3893)\nfinal_res <- final_wflow %>% last_fit(hotel_split, metrics = reg_metrics)\nfinal_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [3749/1251]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [1,251 × 4]> <workflow>" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#asking-for-help", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#asking-for-help", + "title": "1 - Introduction", + "section": "Asking for help", + "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" }, { - "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#test-set-results", - "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#test-set-results", - "title": "5 - Iterative Search", - "section": "Test Set Results", - "text": "Test Set Results\n\n\n\nfinal_res %>% \n collect_predictions() %>% \n cal_plot_regression(\n truth = avg_price_per_room, \n estimate = .pred)\n\nTest set performance:\n\nfinal_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 9.60 Preprocessor1_Model1\n#> 2 rsq standard 0.949 Preprocessor1_Model1\n\n\n\n\n\n\n\n\n\n\n\n\nRecall that resampling predicted the MAE to be 9.601." + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#discord", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#discord", + "title": "1 - Introduction", + "section": "Discord ", + "text": "Discord \n\npos.it/conf-event-portal (login)\nClick on “Join Discord, the virtual networking platform!”\nBrowse Channels -> #workshop-tidymodels-intro" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---setup", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---setup", - "title": "3 - Tuning Hyperparameters", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#section-2", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#section-2", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---data-usage", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---data-usage", - "title": "3 - Tuning Hyperparameters", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#section-3", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#section-3", + "title": "1 - Introduction", + "section": "👀", + "text": "👀\n\n\nhttps://workshops.tidymodels.org" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", - "title": "3 - Tuning Hyperparameters", - "section": "Previously - Feature engineering ", - "text": "Previously - Feature engineering \n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#plan-for-this-workshop", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Plan for this workshop", + "text": "Plan for this workshop\n\nYour data budget\nWhat makes a model\nEvaluating models\nTuning models\n\n\nThis workshop will well-prepare folks going on to the Advanced tidymodels workshop, which will cover feature engineering and much more on hyperparameter tuning." }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-parameters", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#section-4", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#section-4", + "title": "1 - Introduction", + "section": "", + "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free): TODO-ADD-LATER" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "title": "3 - Tuning Hyperparameters", - "section": "Tagging parameters for tuning ", - "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-1", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-1", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", - "title": "3 - Tuning Hyperparameters", - "section": "Optimizing the hash features ", - "text": "Optimizing the hash features \nOur new recipe is:\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\n\nWe will be using a tree-based model in a minute.\n\nThe other categorical predictors are left as-is.\nThat’s why there is no step_dummy()." + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-2024-edition", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-2024-edition", + "title": "1 - Introduction", + "section": "What is machine learning? (2024 edition)", + "text": "What is machine learning? (2024 edition)\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/\n\n\nIn the early 2010s, “Artificial intelligence” (AI) was largely synonymous with what we’ll refer to as “machine learning” in this workshop. In the late 2010s and early 2020s, AI usually referred to deep learning methods. Since the release of ChatGPT in late 2022, “AI” has come to also encompass large language models / generative models." }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-trees", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-trees", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Trees", - "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved and new samples are predicted using a weighted average of the votes of each tree in the ensemble.\n\nWe’ll focus on the popular lightgbm implementation." + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-2", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-2", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands)\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#your-turn", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#your-turn", + "title": "1 - Introduction", + "section": "Your turn", + "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + }, + { + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-tidymodels", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-tidymodels", + "title": "1 - Introduction", + "section": "What is tidymodels? ", + "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.2.0 ──\n#> ✔ broom 1.0.6 ✔ rsample 1.2.1\n#> ✔ dials 1.3.0 ✔ tibble 3.2.1\n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1\n#> ✔ infer 1.0.7 ✔ tune 1.2.1\n#> ✔ modeldata 1.4.0 ✔ workflows 1.1.4\n#> ✔ parsnip 1.2.1 ✔ workflowsets 1.1.0\n#> ✔ purrr 1.0.2 ✔ yardstick 1.3.1\n#> ✔ recipes 1.1.0\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Use tidymodels_prefer() to resolve common conflicts." }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nTBH it is usually not difficult to optimize these models.\n\nOften, there are multiple candidate tuning parameter combinations that have very good results.\n\nTo demonstrate simple concepts, we’ll look at optimizing the number of trees in the ensemble (between 1 and 100) and the learning rate (\\(10^{-5}\\) to \\(10^{-1}\\))." + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\nRoadmap for today\nMinimal version of predictive modeling process\nFeature engineering and tuning as iterative extensions" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters ", - "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-1", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-1", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "title": "3 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-2", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-2", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search", - "title": "3 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-3", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-3", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-1", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-1", - "title": "3 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nIn reality we would probably sample the space more densely:" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-4", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-4", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#iterative-search", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#iterative-search", - "title": "3 - Tuning Hyperparameters", - "section": "Iterative Search", - "text": "Iterative Search\nWe could start with a few points and search the space:" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-5", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-5", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#parameters", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Parameters", - "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-6", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-6", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#different-types-of-grids", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#different-types-of-grids", - "title": "3 - Tuning Hyperparameters", - "section": "Different types of grids ", - "text": "Different types of grids \n\n\n\n\n\n\n\n\n\nSpace-filling designs (SFD) attempt to cover the parameter space without redundant candidates. We recommend these the most." + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-7", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-7", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid", - "title": "3 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\nlgbm_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 4 parameters for tuning\n#> \n#> identifier type object\n#> trees trees nparam[+]\n#> learn_rate learn_rate nparam[+]\n#> agent hash num_terms nparam[+]\n#> company hash num_terms nparam[+]\n\n# Individual functions: \ntrees()\n#> # Trees (quantitative)\n#> Range: [1, 2000]\nlearn_rate()\n#> Learning Rate (quantitative)\n#> Transformer: log-10 [1e-100, Inf]\n#> Range (transformed scale): [-10, -1]\n\n\nA parameter set can be updated (e.g. to change the ranges)." + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#lets-install-some-packages", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"forested\", \"lightgbm\", \"lme4\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid-1", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid-1", - "title": "3 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 7.50e- 6 574 574\n#> 2 84 1.78e- 5 2048 2298\n#> 3 167 5.62e-10 1824 912\n#> 4 250 4.22e- 5 3250 512\n#> 5 334 1.78e- 8 512 2896\n#> 6 417 1.33e- 3 322 1625\n#> 7 500 1 e- 1 1448 1149\n#> 8 584 1 e- 7 1290 256\n#> 9 667 2.37e-10 456 724\n#> 10 750 1.78e- 2 645 322\n#> # ℹ 15 more rows" + "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#our-versions", + "href": "archive/2024-08-posit-conf/intro-01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nR version 4.4.1 (2024-06-14), Quarto (1.6.1)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.3.1\n\n\nbroom\n1.0.6\n\n\nCubist\n0.4.4\n\n\ndials\n1.3.0\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.3\n\n\nembed\n1.1.4\n\n\nfinetune\n1.2.0\n\n\nforested\n0.1.0\n\n\nFormula\n1.2-5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nggplot2\n3.5.1\n\n\nlattice\n0.22-6\n\n\nlightgbm\n4.5.0\n\n\nlme4\n1.1-35.5\n\n\nmodeldata\n1.4.0\n\n\nparallelly\n1.38.0\n\n\nparsnip\n1.2.1\n\n\nplotmo\n3.6.3\n\n\nplotrix\n3.8-4\n\n\nplumber\n1.2.2\n\n\nprobably\n1.0.3\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\npurrr\n1.0.2\n\n\nranger\n0.16.0\n\n\nrecipes\n1.1.0\n\n\nrpart\n4.1.23\n\n\nrpart.plot\n3.1.2\n\n\nrsample\n1.2.1\n\n\nrules\n1.0.2\n\n\nscales\n1.3.0\n\n\nsplines2\n0.5.3\n\n\nstacks\n1.0.5\n\n\ntext2vec\n0.6.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntextrecipes\n1.0.6\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.2.0\n\n\ntidyr\n1.3.1\n\n\ntune\n1.2.1\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.1.4\n\n\nworkflowsets\n1.1.0\n\n\nyardstick\n1.3.1" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn", - "title": "3 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n−+\n03:00" + "objectID": "archive/2024-08-posit-conf/extras-vetiver.html#deploying-a-model", + "href": "archive/2024-08-posit-conf/extras-vetiver.html#deploying-a-model", + "title": "Extras - Model deployment", + "section": "Deploying a model ", + "text": "Deploying a model \nWe have a decision tree, tree_fit, to model whether or not a plot of land in Washington is forested or not.\nHow do we use our model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"forested\")\nv\n#> \n#> ── forested ─ <bundled_workflow> model for deployment \n#> A rpart classification modeling workflow using 18 features\n\nLearn more at https://vetiver.rstudio.com" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", - "title": "3 - Tuning Hyperparameters", - "section": "Create a regular grid ", - "text": "Create a regular grid \n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 4)\n\ngrid\n#> # A tibble: 256 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.0000000001 256 256\n#> 2 667 0.0000000001 256 256\n#> 3 1333 0.0000000001 256 256\n#> 4 2000 0.0000000001 256 256\n#> 5 1 0.0000001 256 256\n#> 6 667 0.0000001 256 256\n#> 7 1333 0.0000001 256 256\n#> 8 2000 0.0000001 256 256\n#> 9 1 0.0001 256 256\n#> 10 667 0.0001 256 256\n#> # ℹ 246 more rows" + "objectID": "archive/2024-08-posit-conf/extras-vetiver.html#deploy-your-model-1", + "href": "archive/2024-08-posit-conf/extras-vetiver.html#deploy-your-model-1", + "title": "Extras - Model deployment", + "section": "Deploy your model ", + "text": "Deploy your model \nHow do we use our model in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 4 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Users/simoncouch/Library/R/arm64/4.4/library/vetiver\n#> ├──/metadata (GET)\n#> ├──/ping (GET)\n#> ├──/predict (POST)\n#> └──/prototype (GET)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-1", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-1", - "title": "3 - Tuning Hyperparameters", + "objectID": "archive/2024-08-posit-conf/extras-vetiver.html#your-turn", + "href": "archive/2024-08-posit-conf/extras-vetiver.html#your-turn", + "title": "Extras - Model deployment", "section": "Your turn", - "text": "Your turn\n\n\nWhat advantage would a regular grid have?" + "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", - "title": "3 - Tuning Hyperparameters", - "section": "Update parameter ranges ", - "text": "Update parameter ranges \n\nlgbm_param <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n update(trees = trees(c(1L, 100L)),\n learn_rate = learn_rate(c(-5, -1)))\n\nset.seed(712)\ngrid <- \n lgbm_param %>% \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.00147 574 574\n#> 2 5 0.00215 2048 2298\n#> 3 9 0.0000215 1824 912\n#> 4 13 0.00316 3250 512\n#> 5 17 0.0001 512 2896\n#> 6 21 0.0147 322 1625\n#> 7 25 0.1 1448 1149\n#> 8 29 0.000215 1290 256\n#> 9 34 0.0000147 456 724\n#> 10 38 0.0464 645 322\n#> # ℹ 15 more rows" + "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#previously---setup", + "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#previously---setup", + "title": "Extras - Effect Encodings", + "section": "Previously - Setup", + "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#the-results", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#the-results", - "title": "3 - Tuning Hyperparameters", - "section": "The results ", - "text": "The results \n\n\ngrid %>% \n ggplot(aes(trees, learn_rate)) +\n geom_point(size = 4) +\n scale_y_log10()\n\n\n\n\n\n\n\n\n\nNote that the learning rates are uniform on the log-10 scale and this shows 2 of 4 dimensions." + "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#previously---data-usage", + "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#previously---data-usage", + "title": "Extras - Effect Encodings", + "section": "Previously - Data Usage", + "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Choosing tuning parameters ", - "text": "Choosing tuning parameters \nLet’s take our previous model and tune more parameters:\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)\n\n# Update the feature hash ranges (log-2 units)\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", + "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", + "title": "Extras - Effect Encodings", + "section": "What do we do with the agent and company data?", + "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-3", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-3", - "title": "3 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE)\n\nlgbm_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = 25,\n # The options below are not required by default\n param_info = lgbm_param, \n control = ctrl,\n metrics = reg_metrics\n )\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" + "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#per-agent-statistics", + "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#per-agent-statistics", + "title": "Extras - Effect Encodings", + "section": "Per-agent statistics", + "text": "Per-agent statistics" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-4", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-4", - "title": "3 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nlgbm_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [3372/377]> Fold01 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,425 × 9]>\n#> 2 <split [3373/376]> Fold02 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 3 <split [3373/376]> Fold03 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 4 <split [3373/376]> Fold04 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 5 <split [3373/376]> Fold05 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 6 <split [3374/375]> Fold06 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,375 × 9]>\n#> 7 <split [3375/374]> Fold07 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,350 × 9]>\n#> 8 <split [3376/373]> Fold08 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 9 <split [3376/373]> Fold09 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 10 <split [3376/373]> Fold10 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>" + "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#what-is-an-effect-encoding", + "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#what-is-an-effect-encoding", + "title": "Extras - Effect Encodings", + "section": "What is an effect encoding?", + "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <fct> <int>\n#> 1 52.7 cynthia_worsley 1\n#> 2 51.8 carlos_bryant 2\n#> 3 53.8 lance_hitchcock 3\n#> 4 51.8 lance_hitchcock 4\n#> 5 46.8 cynthia_worsley 5\n#> 6 54.7 charles_najera 6\n#> 7 46.8 cynthia_worsley 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <dbl> <int>\n#> 1 52.7 88.5 1\n#> 2 51.8 89.5 2\n#> 3 53.8 79.8 3\n#> 4 51.8 79.8 4\n#> 5 46.8 88.5 5\n#> 6 54.7 109. 6\n#> 7 46.8 88.5 7\n\n\nThe agent column is replaced with an estimate of the ADR." }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-results", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-results", - "title": "3 - Tuning Hyperparameters", - "section": "Grid results ", - "text": "Grid results \n\nautoplot(lgbm_res)" + "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#per-agent-statistics-again", + "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#per-agent-statistics-again", + "title": "Extras - Effect Encodings", + "section": "Per-agent statistics again", + "text": "Per-agent statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these means use partial pooling.\nPooling borrows strength across agents and shrinks extreme values towards the mean for agents with very few transations\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for agents with fewer reservations by shrinking the estimate to the overall ADR mean" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results", - "title": "3 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res)\n#> # A tibble: 50 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 298 19 4.15e- 9 222 36 mae standard 53.2 10 0.427 Preprocessor01_Model1\n#> 2 298 19 4.15e- 9 222 36 rsq standard 0.810 10 0.00686 Preprocessor01_Model1\n#> 3 1394 5 5.82e- 6 28 21 mae standard 52.9 10 0.424 Preprocessor02_Model1\n#> 4 1394 5 5.82e- 6 28 21 rsq standard 0.810 10 0.00800 Preprocessor02_Model1\n#> 5 774 12 4.41e- 2 27 95 mae standard 9.77 10 0.155 Preprocessor03_Model1\n#> 6 774 12 4.41e- 2 27 95 rsq standard 0.946 10 0.00341 Preprocessor03_Model1\n#> 7 1342 7 6.84e-10 71 17 mae standard 53.2 10 0.427 Preprocessor04_Model1\n#> 8 1342 7 6.84e-10 71 17 rsq standard 0.811 10 0.00785 Preprocessor04_Model1\n#> 9 669 39 8.62e- 7 141 145 mae standard 53.2 10 0.426 Preprocessor05_Model1\n#> 10 669 39 8.62e- 7 141 145 rsq standard 0.807 10 0.00639 Preprocessor05_Model1\n#> # ℹ 40 more rows" + "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#partial-pooling", + "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#partial-pooling", + "title": "Extras - Effect Encodings", + "section": "Partial pooling", + "text": "Partial pooling" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results-1", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results-1", - "title": "3 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res, summarize = FALSE)\n#> # A tibble: 500 × 10\n#> id trees min_n learn_rate `agent hash` `company hash` .metric .estimator .estimate .config \n#> <chr> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 Fold01 298 19 0.00000000415 222 36 mae standard 51.8 Preprocessor01_Model1\n#> 2 Fold01 298 19 0.00000000415 222 36 rsq standard 0.821 Preprocessor01_Model1\n#> 3 Fold02 298 19 0.00000000415 222 36 mae standard 52.1 Preprocessor01_Model1\n#> 4 Fold02 298 19 0.00000000415 222 36 rsq standard 0.804 Preprocessor01_Model1\n#> 5 Fold03 298 19 0.00000000415 222 36 mae standard 52.2 Preprocessor01_Model1\n#> 6 Fold03 298 19 0.00000000415 222 36 rsq standard 0.786 Preprocessor01_Model1\n#> 7 Fold04 298 19 0.00000000415 222 36 mae standard 51.7 Preprocessor01_Model1\n#> 8 Fold04 298 19 0.00000000415 222 36 rsq standard 0.826 Preprocessor01_Model1\n#> 9 Fold05 298 19 0.00000000415 222 36 mae standard 55.2 Preprocessor01_Model1\n#> 10 Fold05 298 19 0.00000000415 222 36 rsq standard 0.845 Preprocessor01_Model1\n#> # ℹ 490 more rows" + "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#agent-effects", + "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#agent-effects", + "title": "Extras - Effect Encodings", + "section": "Agent effects ", + "text": "Agent effects \n\nlibrary(embed)\n\nhotel_effect_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_lencode_mixed(agent, company, outcome = vars(avg_price_per_room)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", - "title": "3 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \n\nshow_best(lgbm_res, metric = \"rsq\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1890 10 0.0159 115 174 rsq standard 0.948 10 0.00334 Preprocessor12_Model1\n#> 2 774 12 0.0441 27 95 rsq standard 0.946 10 0.00341 Preprocessor03_Model1\n#> 3 1638 36 0.0409 15 120 rsq standard 0.945 10 0.00384 Preprocessor16_Model1\n#> 4 963 23 0.00556 157 13 rsq standard 0.937 10 0.00320 Preprocessor06_Model1\n#> 5 590 5 0.00320 85 73 rsq standard 0.908 10 0.00465 Preprocessor24_Model1" + "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#effect-encoding-results", + "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#effect-encoding-results", + "title": "Extras - Effect Encodings", + "section": "Effect encoding results ", + "text": "Effect encoding results \n\nhotel_effect_wflow <-\n workflow() %>%\n add_model(linear_reg()) %>% \n update_recipe(hotel_effect_rec)\n\nreg_metrics <- metric_set(mae, rsq)\n\nhotel_effect_res <-\n hotel_effect_wflow %>%\n fit_resamples(hotel_rs, metrics = reg_metrics)\n\ncollect_metrics(hotel_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.8 10 0.189 Preprocessor1_Model1\n#> 2 rsq standard 0.870 10 0.00357 Preprocessor1_Model1\n\nSlightly worse but it can handle new agents (if they occur)." }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "title": "3 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nlgbm_best <- select_best(lgbm_res, metric = \"mae\")\nlgbm_best\n#> # A tibble: 1 × 6\n#> trees min_n learn_rate `agent hash` `company hash` .config \n#> <int> <int> <dbl> <int> <int> <chr> \n#> 1 1890 10 0.0159 115 174 Preprocessor12_Model1" + "objectID": "archive/2024-08-posit-conf/advanced-06-wrapping-up.html#your-turn", + "href": "archive/2024-08-posit-conf/advanced-06-wrapping-up.html#your-turn", + "title": "6 - Wrapping up", + "section": "Your turn", + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#checking-calibration", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#checking-calibration", - "title": "3 - Tuning Hyperparameters", - "section": "Checking Calibration ", - "text": "Checking Calibration \n\n\nlibrary(probably)\nlgbm_res %>%\n collect_predictions(\n parameters = lgbm_best\n ) %>%\n cal_plot_regression(\n truth = avg_price_per_room,\n estimate = .pred\n )" + "objectID": "archive/2024-08-posit-conf/advanced-06-wrapping-up.html#resources-to-keep-learning", + "href": "archive/2024-08-posit-conf/advanced-06-wrapping-up.html#resources-to-keep-learning", + "title": "6 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel", - "title": "3 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---setup", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---setup", + "title": "4 - Grid Search via Racing", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, - { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", - "title": "3 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." + { + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---data-usage", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---data-usage", + "title": "4 - Grid Search via Racing", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", - "title": "3 - Tuning Hyperparameters", - "section": "The ‘future’ of parallel processing", - "text": "The ‘future’ of parallel processing\nWe have relied on the foreach package for parallel processing.\nWe will start the transition to using the future package in the upcoming version of the tune package (version 1.3.0).\nThere will be a period of backward compatibility where you can still use foreach with future via the doFuture package. After that, the transition to future will occur.\nOverall, there will be minimal changes to your code." + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---boosting-model", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---boosting-model", + "title": "4 - Grid Search via Racing", + "section": "Previously - Boosting Model ", + "text": "Previously - Boosting Model \n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "title": "3 - Tuning Hyperparameters", - "section": "Early stopping for boosted trees", - "text": "Early stopping for boosted trees\nWe have directly optimized the number of trees as a tuning parameter.\nInstead we could\n\nSet the number of trees to a single large number.\nStop adding trees when performance gets worse.\n\nThis is known as “early stopping” and there is a parameter for that: stop_iter.\nEarly stopping has a potential to decrease the tuning time." + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#first-a-shameless-promotion", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#first-a-shameless-promotion", + "title": "4 - Grid Search via Racing", + "section": "First, a shameless promotion", + "text": "First, a shameless promotion" }, { - "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-2", - "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-2", - "title": "3 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\n\nSet trees = 2000 and tune the stop_iter parameter.\nNote that you will need to regenerate lgbm_param with your new workflow!\n\n\n\n−+\n10:00" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#making-grid-search-more-efficient", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#making-grid-search-more-efficient", + "title": "4 - Grid Search via Racing", + "section": "Making Grid Search More Efficient", + "text": "Making Grid Search More Efficient\nIn the last section, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\nAlso, for some models, we can fit far fewer models than the number that are being evaluated.\n\nFor boosting, a model with X trees can often predict on candidates with less than X trees.\n\nBoth of these methods can lead to enormous speed-ups." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#venue-information", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#venue-information", - "title": "1 - Introduction", - "section": "Venue information", - "text": "Venue information\n\nThere are gender neutral bathrooms located on levels 3, 4, 5, 6 & 7\nA meditation/prayer room is located in 503\n(Mon & Tue 7am - 7pm, and Wed 7am - 5pm)\nA lactation room is located in 509\n(Mon & Tue 7am - 7pm, and Wed 7am - 5pm)" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#model-racing", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#model-racing", + "title": "4 - Grid Search via Racing", + "section": "Model Racing", + "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models but only for a few resamples.\nDetermine which candidates have a low probability of being selected.\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain)\n\nThis can result in fitting a small number of models." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#workshop-policies", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#workshop-policies", - "title": "1 - Introduction", - "section": "Workshop policies", - "text": "Workshop policies\n\nPlease review the posit::conf code of conduct, which applies to all workshops: https://posit.co/code-of-conduct\nCoC site has info on how to report a problem (in person, email, phone)\nPlease do not photograph people wearing red lanyards" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#discarding-candidates", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#discarding-candidates", + "title": "4 - Grid Search via Racing", + "section": "Discarding Candidates", + "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large difference between resamples in the results." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#who-are-you", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML\nYou have used some tidymodels packages\nYou have some experience with evaluating statistical models using resampling techniques" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#resampling-results-non-racing", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#resampling-results-non-racing", + "title": "4 - Grid Search via Racing", + "section": "Resampling Results (Non-Racing)", + "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#who-are-tidymodels", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#who-are-tidymodels", - "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#are-candidates-different", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#are-candidates-different", + "title": "4 - Grid Search via Racing", + "section": "Are Candidates Different?", + "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval for the difference in RMSE is (0.99, 2.8), indicating that candidate number 2 has smaller error." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#asking-for-help", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#asking-for-help", - "title": "1 - Introduction", - "section": "Asking for help", - "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#evaluating-differences-in-candidates", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#evaluating-differences-in-candidates", + "title": "4 - Grid Search via Racing", + "section": "Evaluating Differences in Candidates", + "text": "Evaluating Differences in Candidates\n\n\nWhat if we were to have compared the candidates while we seqeuntially evaluated each resample?\n👉\n\nOne candidate shows superiority when 4 resamples have been evaluated." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#discord", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#discord", - "title": "1 - Introduction", - "section": "Discord ", - "text": "Discord \n\npos.it/conf-event-portal (login)\nClick on “Join Discord, the virtual networking platform!”\nBrowse Channels -> #workshop-tidymodels-advanced" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#interim-analysis-of-results", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#interim-analysis-of-results", + "title": "4 - Grid Search via Racing", + "section": "Interim Analysis of Results", + "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order (so set the seed).\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-2", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-2", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#racing", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#racing", + "title": "4 - Grid Search via Racing", + "section": "Racing ", + "text": "Racing \n\n# Let's use a larger grid\nset.seed(8945)\nlgbm_grid <- \n lgbm_param %>% \n grid_space_filling(size = 50)\n\nlibrary(finetune)\n\nset.seed(9)\nlgbm_race_res <-\n lgbm_wflow %>%\n tune_race_anova(\n resamples = hotel_rs,\n grid = lgbm_grid, \n metrics = reg_metrics\n )\n\nThe syntax and helper functions are extremely similar to those shown for tune_grid()." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-3", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-3", - "title": "1 - Introduction", - "section": "👀", - "text": "👀\n\n\nhttps://workshops.tidymodels.org" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#racing-results", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#racing-results", + "title": "4 - Grid Search via Racing", + "section": "Racing Results ", + "text": "Racing Results \n\nshow_best(lgbm_race_res, metric = \"mae\")\n#> # A tibble: 2 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1347 5 0.0655 66 26 mae standard 9.64 10 0.173 Preprocessor34_Model1\n#> 2 980 8 0.0429 17 135 mae standard 9.76 10 0.164 Preprocessor25_Model1" }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#tentative-plan-for-this-workshop", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#tentative-plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Tentative plan for this workshop", - "text": "Tentative plan for this workshop\n\nFeature engineering with recipes\nModel optimization by tuning\n\nGrid search\nRacing\nIterative methods\n\nExtras (time permitting)\n\nEffect encodings\nA case study" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#racing-results-1", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#racing-results-1", + "title": "4 - Grid Search via Racing", + "section": "Racing Results ", + "text": "Racing Results \n\n\nOnly 171 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_race_res) + \n scale_x_continuous(breaks = pretty_breaks())" }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-4", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-4", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free): TODO-ADD-LATER" + "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#your-turn", + "href": "archive/2024-08-posit-conf/advanced-04-racing.html#your-turn", + "title": "4 - Grid Search via Racing", + "section": "Your turn", + "text": "Your turn\n\nRun tune_race_anova() with a different seed.\nDid you get the same or similar results?\n\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#lets-install-some-packages", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"forested\", \"lightgbm\", \"lme4\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nAlso, you should install the newest version of the dials package (version 1.3.0). To check this, you can run:\n\nrlang::check_installed(\"dials\", version = \"1.3.0\")" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#working-with-our-predictors", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#working-with-our-predictors", + "title": "2 - Feature Engineering", + "section": "Working with our predictors", + "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for linear regression).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#hotel-data", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#hotel-data", - "title": "1 - Introduction", - "section": "Hotel Data ", - "text": "Hotel Data \nWe’ll use data on hotels to predict the cost of a room.\nThe data are in the modeldata package. We’ll sample down the data and refactor some columns:\n\n\n\nlibrary(tidymodels)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#what-is-feature-engineering", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#what-is-feature-engineering", + "title": "2 - Feature Engineering", + "section": "What is feature engineering?", + "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPrincipal component analysis (PCA) feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection (FES) and Applied Machine Learning for Tabular Data (aml4td)." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#hotel-date-columns", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#hotel-date-columns", - "title": "1 - Introduction", - "section": "Hotel date columns", - "text": "Hotel date columns\n\nnames(hotel_rates)\n#> [1] \"avg_price_per_room\" \"lead_time\" \n#> [3] \"stays_in_weekend_nights\" \"stays_in_week_nights\" \n#> [5] \"adults\" \"children\" \n#> [7] \"babies\" \"meal\" \n#> [9] \"country\" \"market_segment\" \n#> [11] \"distribution_channel\" \"is_repeated_guest\" \n#> [13] \"previous_cancellations\" \"previous_bookings_not_canceled\"\n#> [15] \"reserved_room_type\" \"assigned_room_type\" \n#> [17] \"booking_changes\" \"agent\" \n#> [19] \"company\" \"days_in_waiting_list\" \n#> [21] \"customer_type\" \"required_car_parking_spaces\" \n#> [23] \"total_of_special_requests\" \"arrival_date_num\" \n#> [25] \"near_christmas\" \"near_new_years\" \n#> [27] \"historical_adr\"" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#example-dates", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#example-dates", + "title": "2 - Feature Engineering", + "section": "Example: Dates", + "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#data-splitting-strategy", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#data-splitting-strategy", - "title": "1 - Introduction", - "section": "Data splitting strategy", - "text": "Data splitting strategy" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#general-definitions", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#general-definitions", + "title": "2 - Feature Engineering", + "section": "General definitions ", + "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#data-spending", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#data-spending", - "title": "1 - Introduction", - "section": "Data Spending ", - "text": "Data Spending \nLet’s split the data into a training set (75%) and testing set (25%) using stratification:\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#previously---setup", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#previously---setup", + "title": "2 - Feature Engineering", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\n\n# Add another package:\nlibrary(textrecipes)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#your-turn", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#your-turn", - "title": "1 - Introduction", - "section": "Your turn", - "text": "Your turn\nLet’s take some time and investigate the training data. The outcome is avg_price_per_room.\nAre there any interesting characteristics of the data?\n\n\n\n−+\n10:00" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#previously---data-usage", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#previously---data-usage", + "title": "2 - Feature Engineering", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\n\nWe’ll go from here and create a set of resamples to use for model assessments." }, { - "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#our-versions", - "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nR version 4.4.1 (2024-06-14), Quarto (1.6.1)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.3.1\n\n\nbroom\n1.0.6\n\n\nCubist\n0.4.4\n\n\ndials\n1.3.0\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.3\n\n\nembed\n1.1.4\n\n\nfinetune\n1.2.0\n\n\nFormula\n1.2-5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nggplot2\n3.5.1\n\n\nlattice\n0.22-6\n\n\nlightgbm\n4.5.0\n\n\nlme4\n1.1-35.5\n\n\nmodeldata\n1.4.0\n\n\nparallelly\n1.38.0\n\n\nparsnip\n1.2.1\n\n\nplotmo\n3.6.3\n\n\nplotrix\n3.8-4\n\n\nplumber\n1.2.2\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nprobably\n1.0.3\n\n\npurrr\n1.0.2\n\n\nrecipes\n1.1.0\n\n\nrsample\n1.2.1\n\n\nrules\n1.0.2\n\n\nscales\n1.3.0\n\n\nsplines2\n0.5.3\n\n\nstacks\n1.0.5\n\n\ntext2vec\n0.6.4\n\n\ntextrecipes\n1.0.6\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.2.0\n\n\ntidyr\n1.3.1\n\n\ntune\n1.2.1\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.1.4\n\n\nworkflowsets\n1.1.0\n\n\nyardstick\n1.3.1" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#resampling-strategy", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#resampling-strategy", + "title": "2 - Feature Engineering", + "section": "Resampling Strategy", + "text": "Resampling Strategy" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#parámetros-de-afinamiento", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#parámetros-de-afinamiento", - "title": "5 - Afinando modelos", - "section": "Parámetros de afinamiento", - "text": "Parámetros de afinamiento\nAlgunos valores de los parametros de los modelos o de los preprocesadores no se pueden estimar directamente desde los datos\n\nPor ejemplo:\n\nLa profundidad de los árboles de decisión\nLa cantidad de vecinos en un modelo de vecinos K-nearest" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#resampling-strategy-1", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#resampling-strategy-1", + "title": "2 - Feature Engineering", + "section": "Resampling Strategy ", + "text": "Resampling Strategy \nWe’ll use simple 10-fold cross-validation (stratified sampling):\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)\nhotel_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [3372/377]> Fold01\n#> 2 <split [3373/376]> Fold02\n#> 3 <split [3373/376]> Fold03\n#> 4 <split [3373/376]> Fold04\n#> 5 <split [3373/376]> Fold05\n#> 6 <split [3374/375]> Fold06\n#> 7 <split [3375/374]> Fold07\n#> 8 <split [3376/373]> Fold08\n#> 9 <split [3376/373]> Fold09\n#> 10 <split [3376/373]> Fold10" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#optimiza-los-parámetros-de-afinamiento", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#optimiza-los-parámetros-de-afinamiento", - "title": "5 - Afinando modelos", - "section": "Optimiza los parámetros de afinamiento", - "text": "Optimiza los parámetros de afinamiento\n\nPrueba valores diferentes para medir la calidad del modelo\n\n\n\nEncuentra buenos valores para los parametros\n\n\n\n\nUna vez los valores de los parametros han sido determinados, el modelo final se puede ajustar usando todos los datos en el set de entrenamiento" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", + "title": "2 - Feature Engineering", + "section": "Prepare your data for modeling ", + "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of preprocessing and feature engineering steps.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#optimiza-los-parámetros-de-afinamiento-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#optimiza-los-parámetros-de-afinamiento-1", - "title": "5 - Afinando modelos", - "section": "Optimiza los parámetros de afinamiento", - "text": "Optimiza los parámetros de afinamiento\nHay dos estrategias principales para optimizar:\n\n\nBusqueda de cuadrícula (Grid search) 💠 que prueba un set de valores pre-elejidos\nBusqueda iterativa 🌀 que sugiere nuevos valores del parametro para probar" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#a-first-recipe", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#a-first-recipe", + "title": "2 - Feature Engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#especificando-los-parametros-de-afinamiento", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#especificando-los-parametros-de-afinamiento", - "title": "5 - Afinando modelos", - "section": "Especificando los parametros de afinamiento", - "text": "Especificando los parametros de afinamiento\nTomemos nueso modelo de bosque aleatorio y tratemos de encontrar el número mínimo de datos min_n\n\narbol_espec <- rand_forest(min_n = tune()) %>% \n set_mode(\"classification\")\n\narbol_flujo <- workflow(propina ~ ., arbol_espec)\narbol_flujo\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#a-first-recipe-1", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#a-first-recipe-1", + "title": "2 - Feature Engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nsummary(hotel_rec)\n#> # A tibble: 27 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 lead_time <chr [2]> predictor original\n#> 2 stays_in_weekend_nights <chr [2]> predictor original\n#> 3 stays_in_week_nights <chr [2]> predictor original\n#> 4 adults <chr [2]> predictor original\n#> 5 children <chr [2]> predictor original\n#> 6 babies <chr [2]> predictor original\n#> 7 meal <chr [3]> predictor original\n#> 8 country <chr [3]> predictor original\n#> 9 market_segment <chr [3]> predictor original\n#> 10 distribution_channel <chr [3]> predictor original\n#> # ℹ 17 more rows\n\nThe type column contains information on the variables" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#try-out-multiple-values", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#try-out-multiple-values", - "title": "5 - Afinando modelos", - "section": "Try out multiple values", - "text": "Try out multiple values" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn", + "title": "2 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\nWhat do you think are in the type vectors for the lead_time and country columns?\n\n\n\n−+\n02:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#prueba-varios-valores", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#prueba-varios-valores", - "title": "5 - Afinando modelos", - "section": "Prueba varios valores", - "text": "Prueba varios valores\ntune_grid() funciona parecido a fit_resamples() pero puede calcular multiples valores del parámetro:\n\nset.seed(22)\nrf_res <- tune_grid(\n arbol_flujo,\n taxi_plieges,\n grid = 5\n)" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#create-indicator-variables", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#create-indicator-variables", + "title": "2 - Feature Engineering", + "section": "Create indicator variables ", + "text": "Create indicator variables \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns.\nstep_dummy() records the levels of the categorical predictors in the training set." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#compara-los-resultados", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#compara-los-resultados", - "title": "5 - Afinando modelos", - "section": "Compara los resultados", - "text": "Compara los resultados\nInspecciona los resultados, y selecciona los mejores parámetros:\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 33 roc_auc binary 0.653 10 0.0159 Preprocessor1_Model1\n#> 2 31 roc_auc binary 0.650 10 0.0167 Preprocessor1_Model3\n#> 3 21 roc_auc binary 0.646 10 0.0165 Preprocessor1_Model4\n#> 4 13 roc_auc binary 0.646 10 0.0143 Preprocessor1_Model5\n#> 5 6 roc_auc binary 0.631 10 0.0140 Preprocessor1_Model2\n\nparametro_mejor <- select_best(rf_res)\nparametro_mejor\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 33 Preprocessor1_Model1\n\nTambién puedes usar collect_metrics() y autoplot()" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#filter-out-constant-columns", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#filter-out-constant-columns", + "title": "2 - Feature Engineering", + "section": "Filter out constant columns ", + "text": "Filter out constant columns \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#el-ajuste-final", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#el-ajuste-final", - "title": "5 - Afinando modelos", - "section": "El ajuste final", - "text": "El ajuste final\n\narbol_flujo <- finalize_workflow(arbol_flujo, parametro_mejor)\n\najuste_final <- last_fit(arbol_flujo, taxi_separar) \n\ncollect_metrics(ajuste_final)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.912 Preprocessor1_Model1\n#> 2 roc_auc binary 0.636 Preprocessor1_Model1" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#normalization", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#normalization", + "title": "2 - Feature Engineering", + "section": "Normalization ", + "text": "Normalization \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#tu-turno", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#tu-turno", - "title": "5 - Afinando modelos", - "section": "Tu turno", - "text": "Tu turno\n\nModifica tu flujo de modelo para usar por lo menos un parámetro\nUsa busqueda de cuadricula para encontrar los mejores parametros\n\n\n\n−+\n05:00" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#reduce-correlation", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#reduce-correlation", + "title": "2 - Feature Engineering", + "section": "Reduce correlation ", + "text": "Reduce correlation \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno", - "title": "3 - Las partes de un modelo", - "section": "Tu turno", - "text": "Tu turno\n\n¿Como ajustar un modelo linear en R?\n¿De cuantas maneras sabes como ajustar este tipo de modelo?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm para regresión linear\nglm para regresión linear generalizada\nglmnet para regresión regularizada\nkeras para regresión dentro de Tensorflow\nstan para regresión bayensiana\nspark para datos “grandes”" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps", + "title": "2 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo", - "title": "3 - Las partes de un modelo", - "section": "Para especificar un modelo ", - "text": "Para especificar un modelo \n\n\n\n\nElije un modelo\nEspecifica el “motor”\nEstablece el modo" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-1", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-1", + "title": "2 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = vars(avg_price_per_room))\n\n\nA fancy machine learning supervised dimension reduction technique called UMAP…\n\nNote that this uses the outcome, and it is from an extension package" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-1", - "title": "3 - Las partes de un modelo", - "section": "Para especificar un modelo ", - "text": "Para especificar un modelo \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-2", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-2", + "title": "2 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-2", - "title": "3 - Las partes de un modelo", - "section": "Para especificar un modelo ", - "text": "Para especificar un modelo \n\n\n\nElije un modelo\nEspecifica el “motor”\nEstablece el modo" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn-1", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn-1", + "title": "2 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\n\nCreate a recipe() for the hotel data to:\n\nuse a Yeo-Johnson (YJ) transformation on lead_time\nconvert factors to indicator variables\nremove zero-variance variables\nadd the spline technique shown above\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-3", - "title": "3 - Las partes de un modelo", - "section": "Para especificar un modelo ", - "text": "Para especificar un modelo \n\nlogistic_reg() %>%\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#minimal-recipe", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#minimal-recipe", + "title": "2 - Feature Engineering", + "section": "Minimal recipe ", + "text": "Minimal recipe \n\nhotel_indicators <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-4", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-4", - "title": "3 - Las partes de un modelo", - "section": "Para especificar un modelo ", - "text": "Para especificar un modelo \n\nlogistic_reg() %>%\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#measuring-performance", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#measuring-performance", + "title": "2 - Feature Engineering", + "section": "Measuring Performance ", + "text": "Measuring Performance \nWe’ll compute two measures: mean absolute error and the coefficient of determination (a.k.a \\(R^2\\)).\n\\[\\begin{align}\nMAE &= \\frac{1}{n}\\sum_{i=1}^n |y_i - \\hat{y}_i| \\notag \\\\\nR^2 &= cor(y_i, \\hat{y}_i)^2\n\\end{align}\\]\nThe focus will be on MAE for parameter optimization. We’ll use a metric set to compute these:\n\nreg_metrics <- metric_set(mae, rsq)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-5", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-5", - "title": "3 - Las partes de un modelo", - "section": "Para especificar un modelo ", - "text": "Para especificar un modelo \n\n\n\nElije un modelo\nEspecifica el “motor”\nEstablece el modo" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#using-a-workflow", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#using-a-workflow", + "title": "2 - Feature Engineering", + "section": "Using a workflow ", + "text": "Using a workflow \n\nset.seed(9)\n\nhotel_lm_wflow <-\n workflow() %>%\n add_recipe(hotel_indicators) %>%\n add_model(linear_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nhotel_lm_res <-\n hotel_lm_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_lm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.6 10 0.214 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00339 Preprocessor1_Model1" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-6", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-6", - "title": "3 - Las partes de un modelo", - "section": "Para especificar un modelo ", - "text": "Para especificar un modelo \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn-2", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn-2", + "title": "2 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-7", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-7", - "title": "3 - Las partes de un modelo", - "section": "Para especificar un modelo ", - "text": "Para especificar un modelo \n\ndecision_tree() %>% \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nEl listado de modelos se encuentra aquí: https://www.tidymodels.org/find/parsnip/" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#holdout-predictions", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#holdout-predictions", + "title": "2 - Feature Engineering", + "section": "Holdout predictions ", + "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nlm_cv_pred <- collect_predictions(hotel_lm_res)\nlm_cv_pred %>% print(n = 7)\n#> # A tibble: 3,749 × 5\n#> .pred id .row avg_price_per_room .config \n#> <dbl> <chr> <int> <dbl> <chr> \n#> 1 75.1 Fold01 20 40 Preprocessor1_Model1\n#> 2 49.3 Fold01 28 54 Preprocessor1_Model1\n#> 3 64.9 Fold01 45 50 Preprocessor1_Model1\n#> 4 52.8 Fold01 49 42 Preprocessor1_Model1\n#> 5 48.6 Fold01 61 49 Preprocessor1_Model1\n#> 6 29.8 Fold01 66 40 Preprocessor1_Model1\n#> 7 36.9 Fold01 88 49 Preprocessor1_Model1\n#> # ℹ 3,742 more rows" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#to-specify-a-model", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#to-specify-a-model", - "title": "3 - Las partes de un modelo", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nElije un modelo\nEspecifica el “motor”\nEstablece el modo" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#calibration-plot", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#calibration-plot", + "title": "2 - Feature Engineering", + "section": "Calibration Plot ", + "text": "Calibration Plot \n\nlibrary(probably)\n\ncal_plot_regression(hotel_lm_res)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-1", - "title": "3 - Las partes de un modelo", - "section": "Tu turno", - "text": "Tu turno\n\nCorre el “chunk” arbol_espec en tu archivo .qmd.\nCambia el código para que utilize una regresión linear\n\nEl listado de modelos se encuentra aquí: https://www.tidymodels.org/find/parsnip/\n\n\nReto: Edita el código para usar otro tipo de modelo. Por ejemplo, trata de usar el árbol disponible dentro del paquete partykit, o trata un modelo de tipo diferente\n\n\n\n−+\n05:00" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", + "title": "2 - Feature Engineering", + "section": "What do we do with the agent and company data?", + "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 unique companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor (in the extra materials)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#los-modelos-que-usaremos-hoy", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#los-modelos-que-usaremos-hoy", - "title": "3 - Las partes de un modelo", - "section": "Los modelos que usaremos hoy", - "text": "Los modelos que usaremos hoy\n\nRegresión logística\nArboles de decisión" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#per-agent-statistics", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#per-agent-statistics", + "title": "2 - Feature Engineering", + "section": "Per-agent statistics", + "text": "Per-agent statistics" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística", - "title": "3 - Las partes de un modelo", - "section": "Regresión logística", - "text": "Regresión logística" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#collapsing-factor-levels", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#collapsing-factor-levels", + "title": "2 - Feature Engineering", + "section": "Collapsing factor levels ", + "text": "Collapsing factor levels \nThere is a recipe step that will redefine factor levels based on their frequency in the training set:\n\nhotel_other_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_other(agent, threshold = 0.001) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nUsing this code, 34 agents (out of 98) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística-1", - "title": "3 - Las partes de un modelo", - "section": "Regresión logística", - "text": "Regresión logística" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#does-othering-help", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#does-othering-help", + "title": "2 - Feature Engineering", + "section": "Does othering help? ", + "text": "Does othering help? \n\nhotel_other_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hotel_other_rec)\n\nhotel_other_res <-\n hotel_other_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.213 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00341 Preprocessor1_Model1\n\nAbout the same MAE and much faster to complete.\nNow let’s look at a more sophisticated tool called effect feature hashing." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística-2", - "title": "3 - Las partes de un modelo", - "section": "Regresión logística", - "text": "Regresión logística\n\n\n\n\n\n\n\n\n\n\n\n\n\nEl logit del la probabilidad del resultado\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nCalcular la línea de sigmoideo que separan las dos clases" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing", + "title": "2 - Feature Engineering", + "section": "Feature Hashing", + "text": "Feature Hashing\nBetween agent and company, simple dummy variables would create 198 new columns (that are mostly zeros).\nAnother option is to have a binary indicator that combines some levels of these variables.\nFeature hashing (for more see FES, SMLTAR, TMwR, and aml4td):\n\nuses the character values of the levels\nconverts them to integer hash values\nuses the integers to assign them to a specific indicator column." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión", - "title": "3 - Las partes de un modelo", - "section": "Arboles de decisión", - "text": "Arboles de decisión" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-1", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-1", + "title": "2 - Feature Engineering", + "section": "Feature Hashing", + "text": "Feature Hashing\nSuppose we want to use 32 indicator variables for agent.\nFor a agent with value “Max_Kuhn”, a hashing function converts it to an integer (say 210397726).\nTo assign it to one of the 32 columns, we would use modular arithmetic to assign it to a column:\n\n# For \"Max_Kuhn\" put a '1' in column: \n210397726 %% 32\n#> [1] 30\n\nHash functions are meant to emulate randomness." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión-1", - "title": "3 - Las partes de un modelo", - "section": "Arboles de decisión", - "text": "Arboles de decisión\n\n\n\n\n\n\n\n\n\n\n\n\n\nEl resultado es decidido usando una serie de decisiones basados en las variables predictivas\nPrimero, el arbol crece hasta que se llega a una decision\nDespués, el arbol es recortado para que no sea tan complejo" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-pros", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-pros", + "title": "2 - Feature Engineering", + "section": "Feature Hashing Pros", + "text": "Feature Hashing Pros\n\nThe procedure will automatically work on new values of the predictors.\nIt is fast.\n“Signed” hashes add a sign to help avoid aliasing." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión-2", - "title": "3 - Las partes de un modelo", - "section": "Arboles de decisión", - "text": "Arboles de decisión" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-cons", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-cons", + "title": "2 - Feature Engineering", + "section": "Feature Hashing Cons", + "text": "Feature Hashing Cons\n\nThere is no real logic behind which factor levels are combined.\nWe don’t know how many columns to add (more in the next section).\nSome columns may have all zeros.\nIf a indicator column is important to the model, we can’t easily determine why.\n\n\nThe signed hash make it slightly more possible to differentiate between confounded levels" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#todos-los-modelos-están-equivocados-pero-algunos-pueden-ser-utíl", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#todos-los-modelos-están-equivocados-pero-algunos-pueden-ser-utíl", - "title": "3 - Las partes de un modelo", - "section": "Todos los modelos están equivocados, pero algunos pueden ser utíl", - "text": "Todos los modelos están equivocados, pero algunos pueden ser utíl\n\n\nRegresión logística\n\n\n\n\n\n\n\n\n\n\nArboles de decisión" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes", + "title": "2 - Feature Engineering", + "section": "Feature Hashing in recipes ", + "text": "Feature Hashing in recipes \nThe textrecipes package has a step that can be added to the recipe:\n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nhotel_hash_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hash_rec)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#los-flujos-de-trabajo-combinan-el-preprocesamiento-y-el-modelo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#los-flujos-de-trabajo-combinan-el-preprocesamiento-y-el-modelo", - "title": "3 - Las partes de un modelo", - "section": "Los flujos de trabajo combinan el “preprocesamiento” y el modelo", - "text": "Los flujos de trabajo combinan el “preprocesamiento” y el modelo\n\n\n\n\n\nflowchart LR\n dt[Datos]\n pr[Predictores]\n subgraph fl[Flujo del modelo]\n pca[PCA]\n ls[Menos cuadrados]\n pca-->ls\n end\n dt-->fl\n pr-->fl\n ft[Modelo ajustado]\n fl-->ft\n style fl fill:#fff,stroke:#666,color:#000\n\n\n\n\n\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", + "title": "2 - Feature Engineering", + "section": "Feature Hashing in recipes ", + "text": "Feature Hashing in recipes \n\nhotel_hash_res <-\n hotel_hash_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_hash_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.239 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00324 Preprocessor1_Model1\n\nAbout the same performance but now we can handle new values." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#que-no-está-bien-en-este-flujo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#que-no-está-bien-en-este-flujo", - "title": "3 - Las partes de un modelo", - "section": "¿Que no está bien en este flujo?", - "text": "¿Que no está bien en este flujo?\n\n\n\n\n\nflowchart LR\n dt[Datos]\n pr[Predictores]\n pca[PCA]\n subgraph fl[Flujo del modelo]\n ls[Menos cuadrados]\n end\n pca-->fl\n dt-->pca\n pr-->pca\n ft[Modelo ajustado]\n fl-->ft\n style fl fill:#fff,stroke:#666,color:#000" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#debugging-a-recipe", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#debugging-a-recipe", + "title": "2 - Feature Engineering", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. hash_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." }, - { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#por-que-utilizar-workflow", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#por-que-utilizar-workflow", - "title": "3 - Las partes de un modelo", - "section": "¿Por que utilizar workflow()? ", - "text": "¿Por que utilizar workflow()? \n\n\nworkflow maneja nuevos niveles factoriales mejor que las herramientas regulares de R\n\n\n\n\nPuedes utilizar “preprocesadores” que no son formulas\n\n\n\n\nAyudan a organizar un proyecto que utiliza varios modelos\n\n\n\n\nAún más importante, un flujo de workflow encaja todo el processo de modelamiento, desde la creación (fit()) del modelo, hasta el uso del modelo (predict())\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" + { + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#example", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#example", + "title": "2 - Feature Engineering", + "section": "Example ", + "text": "Example \n\nhash_rec_fit <- prep(hash_rec)\n\n# Get the transformation coefficient\ntidy(hash_rec_fit, number = 1)\n\n# Get the processed data\nbake(hash_rec_fit, hotel_train %>% slice(1:3), contains(\"_agent_\"))" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-1", - "title": "3 - Las partes de un modelo", - "section": "El flujo de trabajo de un modelo ", - "text": "El flujo de trabajo de un modelo \n\narbol_espec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\narbol_espec %>% \n fit(propina ~ ., data = taxi_entrenar) \n#> parsnip model object\n#> \n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 si (0.92300000 0.07700000) \n#> 2) distancia>=14.12 2041 68 si (0.96668300 0.03331700) *\n#> 3) distancia< 14.12 5959 548 si (0.90803826 0.09196174) \n#> 6) distancia< 5.275 5419 450 si (0.91695885 0.08304115) *\n#> 7) distancia>=5.275 540 98 si (0.81851852 0.18148148) \n#> 14) compania=Chicago Independents,City Service,otra,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc 478 68 si (0.85774059 0.14225941) *\n#> 15) compania=Flash Cab 62 30 si (0.51612903 0.48387097) \n#> 30) dia=Jue 12 2 si (0.83333333 0.16666667) *\n#> 31) dia=Lun,Mar,Mie,Vie,Sab,Dom 50 22 no (0.44000000 0.56000000) \n#> 62) distancia>=11.77 14 4 si (0.71428571 0.28571429) *\n#> 63) distancia< 11.77 36 12 no (0.33333333 0.66666667) *" + "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#more-on-recipes", + "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#more-on-recipes", + "title": "2 - Feature Engineering", + "section": "More on recipes", + "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-2", - "title": "3 - Las partes de un modelo", - "section": "El flujo de trabajo de un modelo ", - "text": "El flujo de trabajo de un modelo \n\narbol_espec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\nworkflow() %>%\n add_formula(propina ~ .) %>%\n add_model(arbol_espec) %>%\n fit(data = taxi_entrenar) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 si (0.92300000 0.07700000) \n#> 2) distancia>=14.12 2041 68 si (0.96668300 0.03331700) *\n#> 3) distancia< 14.12 5959 548 si (0.90803826 0.09196174) \n#> 6) distancia< 5.275 5419 450 si (0.91695885 0.08304115) *\n#> 7) distancia>=5.275 540 98 si (0.81851852 0.18148148) \n#> 14) compania=Chicago Independents,City Service,otra,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc 478 68 si (0.85774059 0.14225941) *\n#> 15) compania=Flash Cab 62 30 si (0.51612903 0.48387097) \n#> 30) dia=Jue 12 2 si (0.83333333 0.16666667) *\n#> 31) dia=Lun,Mar,Mie,Vie,Sab,Dom 50 22 no (0.44000000 0.56000000) \n#> 62) distancia>=11.77 14 4 si (0.71428571 0.28571429) *\n#> 63) distancia< 11.77 36 12 no (0.33333333 0.66666667) *" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-06-para-terminar.html#tu-turno", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-06-para-terminar.html#tu-turno", + "title": "6 - Para terminar", + "section": "Tu turno", + "text": "Tu turno\n\n¿Que fue algo que te sorprendio aprender?\n¿Cual será lo que aprendiste que planeas usar?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-3", - "title": "3 - Las partes de un modelo", - "section": "El flujo de trabajo de un modelo ", - "text": "El flujo de trabajo de un modelo \n\narbol_espec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\nworkflow(propina ~ ., arbol_espec) %>% \n fit(data = taxi_entrenar) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 si (0.92300000 0.07700000) \n#> 2) distancia>=14.12 2041 68 si (0.96668300 0.03331700) *\n#> 3) distancia< 14.12 5959 548 si (0.90803826 0.09196174) \n#> 6) distancia< 5.275 5419 450 si (0.91695885 0.08304115) *\n#> 7) distancia>=5.275 540 98 si (0.81851852 0.18148148) \n#> 14) compania=Chicago Independents,City Service,otra,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc 478 68 si (0.85774059 0.14225941) *\n#> 15) compania=Flash Cab 62 30 si (0.51612903 0.48387097) \n#> 30) dia=Jue 12 2 si (0.83333333 0.16666667) *\n#> 31) dia=Lun,Mar,Mie,Vie,Sab,Dom 50 22 no (0.44000000 0.56000000) \n#> 62) distancia>=11.77 14 4 si (0.71428571 0.28571429) *\n#> 63) distancia< 11.77 36 12 no (0.33333333 0.66666667) *" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-06-para-terminar.html#recursos-para-seguir-aprendiendo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-06-para-terminar.html#recursos-para-seguir-aprendiendo", + "title": "6 - Para terminar", + "section": "Recursos para seguir aprendiendo", + "text": "Recursos para seguir aprendiendo\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-2", - "title": "3 - Las partes de un modelo", - "section": "Tu turno", - "text": "Tu turno\n\nCorre el “chunk” arbol_flujo en tu archivo .qmd.\nEdita el codigo para crear un flujo con un modelo que tú elijas\n\nReto: ¿Que otros “preprocesadores”, aparte de formulas, podemos usar en workflow?\n\n\n\n−+\n05:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#viendo-las-predicciones", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#viendo-las-predicciones", + "title": "4 - Evaluar modelos", + "section": "Viendo las predicciones", + "text": "Viendo las predicciones\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n relocate(propina, .pred_class, .pred_si, .pred_no)\n#> # A tibble: 8,000 × 10\n#> propina .pred_class .pred_si .pred_no distancia compania local dia mes \n#> <fct> <fct> <dbl> <dbl> <dbl> <fct> <fct> <fct> <fct>\n#> 1 si si 0.967 0.0333 17.2 Chicago In… no Jue Feb \n#> 2 si si 0.935 0.0646 0.88 City Servi… si Jue Mar \n#> 3 si si 0.967 0.0333 18.1 otra no Lun Feb \n#> 4 si si 0.949 0.0507 12.2 Chicago In… no Dom Mar \n#> 5 si si 0.821 0.179 0.94 Sun Taxi si Sab Abr \n#> 6 si si 0.967 0.0333 17.5 Flash Cab no Vie Mar \n#> 7 si si 0.967 0.0333 17.7 otra no Dom Ene \n#> 8 si si 0.938 0.0616 1.85 Taxicab In… no Vie Abr \n#> 9 si si 0.938 0.0616 0.53 Sun Taxi no Mar Mar \n#> 10 si si 0.931 0.0694 6.65 Taxicab In… no Dom Abr \n#> # ℹ 7,990 more rows\n#> # ℹ 1 more variable: hora <int>" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#predecir-usando-tu-modelo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#predecir-usando-tu-modelo", - "title": "3 - Las partes de un modelo", - "section": "Predecir usando tu modelo ", - "text": "Predecir usando tu modelo \n¿Como usar su nuevo modelo arbol_flujo?\n\narbol_espec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\narbol_flujo <-\n workflow(propina ~ ., arbol_espec) %>% \n fit(data = taxi_entrenar)" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión", + "title": "4 - Evaluar modelos", + "section": "Matriz de confusión ", + "text": "Matriz de confusión" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-3", - "title": "3 - Las partes de un modelo", - "section": "Tu turno", - "text": "Tu turno\n\nCorre:\npredict(arbol_flujo, new_data = taxi_prueba)\n¿Que resultado te dio?\n\n\n\n−+\n03:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión-1", + "title": "4 - Evaluar modelos", + "section": "Matriz de confusión ", + "text": "Matriz de confusión \n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n conf_mat(truth = propina, estimate = .pred_class)\n#> Truth\n#> Prediction si no\n#> si 7341 536\n#> no 43 80" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-4", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-4", - "title": "3 - Las partes de un modelo", - "section": "Tu turno", - "text": "Tu turno\n\nCorre:\naugment(arbol_flujo, new_data = taxi_prueba)\n¿Que resultado te dio?\n\n\n\n−+\n03:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión-2", + "title": "4 - Evaluar modelos", + "section": "Matriz de confusión ", + "text": "Matriz de confusión \n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n conf_mat(truth = propina, estimate = .pred_class) %>%\n autoplot(type = \"heatmap\")" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo", - "title": "3 - Las partes de un modelo", - "section": "Entiendedo tú modelo ", - "text": "Entiendedo tú modelo \n¿Como podemos entender nuestro nuevo modelo arbol_flujo?" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo", + "title": "4 - Evaluar modelos", + "section": "Mediciones de la calidad del modelo ", + "text": "Mediciones de la calidad del modelo \n\n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n accuracy(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo-1", - "title": "3 - Las partes de un modelo", - "section": "Entiendedo tú modelo ", - "text": "Entiendedo tú modelo \n¿Como podemos entender nuestro nuevo modelo arbol_flujo?\n\nlibrary(rpart.plot)\narbol_flujo %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nPuedes extraer varios componentes de tu flujo usando las funciones que empiezan con extract_*()\n\n⚠️ ¡Nunca trate de predecir usando los componentes extraídos!\n\nroundint = FALSE is only to quiet a warning" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#el-riesgo-de-concentrarse-en-la-exactitud", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#el-riesgo-de-concentrarse-en-la-exactitud", + "title": "4 - Evaluar modelos", + "section": "El riesgo de concentrarse en la exactitud ", + "text": "El riesgo de concentrarse en la exactitud \nHay que tener cuidado utilizando exactitud (accuracy()) ya que nos puede dar “buenos” resultado se predecimos con datos que no están balanceados\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n mutate(.pred_class = factor(\"si\", levels = c(\"si\", \"no\"))) %>%\n accuracy(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.923" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo-2", - "title": "3 - Las partes de un modelo", - "section": "Entiendedo tú modelo ", - "text": "Entiendedo tú modelo \n¿Como podemos entender nuestro nuevo modelo arbol_flujo?\n\nPuedes usar tu flujo ajustado para obtener explicaciones del modelo o de las predicciones:\n\n\n\nEl paquete vip puede dar la importancia de cada variable\n\n\n\n\nPara obtener explicaciones del modelo, puede usar el paquete DALEXtra\n\n\n\nPara aprender más: https://www.tmwr.org/explain.html" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-1", + "title": "4 - Evaluar modelos", + "section": "Mediciones de la calidad del modelo ", + "text": "Mediciones de la calidad del modelo \n\n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n sensitivity(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.994" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-5", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-5", - "title": "3 - Las partes de un modelo", - "section": "Tu turno", - "text": "Tu turno\n\n\nExtrae el “motor” del modelo de tu flujo y examínalo\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-2", + "title": "4 - Evaluar modelos", + "section": "Mediciones de la calidad del modelo ", + "text": "Mediciones de la calidad del modelo \n\n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n sensitivity(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.994\n\n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n specificity(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.130" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#expectativas-del-taller---donde-estamos", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#expectativas-del-taller---donde-estamos", - "title": "3 - Las partes de un modelo", - "section": "Expectativas del taller - Donde estamos", - "text": "Expectativas del taller - Donde estamos\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#fff,stroke:#eee,color:#ddd\n tr --> dt\n lg[Regresión\\nlogística]\n style lg fill:#fff,stroke:#eee,color:#ddd\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#fff,stroke:#eee,color:#ddd\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#fff,stroke:#eee,color:#ddd\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#fff,stroke:#eee,color:#ddd\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#fff,stroke:#eee,color:#ddd\n fm --> vm\n ts --> vm\n\n\n\n\n\n\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-3", + "title": "4 - Evaluar modelos", + "section": "Mediciones de la calidad del modelo ", + "text": "Mediciones de la calidad del modelo \nPara combinar multiples cálculos en una tabla, usa metric_set()\n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n taxi_metrics(truth = propina, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n#> 2 specificity binary 0.130\n#> 3 sensitivity binary 0.994" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#normas-del-taller", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#normas-del-taller", - "title": "1 - Introducción", - "section": "Normas del taller", - "text": "Normas del taller\n\nCódigo de conducta: TODO-ADD-LATER" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-4", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-4", + "title": "4 - Evaluar modelos", + "section": "Mediciones de la calidad del modelo ", + "text": "Mediciones de la calidad del modelo \n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n group_by(local) %>%\n taxi_metrics(truth = propina, estimate = .pred_class)\n#> # A tibble: 6 × 4\n#> local .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 si accuracy binary 0.898\n#> 2 no accuracy binary 0.935\n#> 3 si specificity binary 0.169\n#> 4 no specificity binary 0.116\n#> 5 si sensitivity binary 0.987\n#> 6 no sensitivity binary 0.996" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#quien-eres", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#quien-eres", - "title": "1 - Introducción", - "section": "Quien eres?", - "text": "Quien eres?\n\nSabe utilizar la “pipa” de magritr (%>%) o R (|>)\nConoce las funciones de dplyr, tidyr y ggplot2\nEntiende conceptos estadísticos básicos\nNo necesitará ser experto en modelaje o aprendizaje automático" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#resultados-de-dos-clases", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#resultados-de-dos-clases", + "title": "4 - Evaluar modelos", + "section": "Resultados de dos clases", + "text": "Resultados de dos clases\nEstas métricas asumen que sabemos cual es límite para convertir probabilidades de predicción “suaves” a prediciones de clase “duras” . . .\n¿Es bueno un límite de 50%?\n¿Que pasaría si lo cambiamos a 80%?\n\nsensibilidad ⬇️, especificidad ⬆️\n\n\n¿Y si lo cambiamos a 20%?\n\nsensibilidad ⬆️, especificidad ⬇️" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#el-equipo-tidymodels", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#el-equipo-tidymodels", - "title": "1 - Introducción", - "section": "El equipo Tidymodels", - "text": "El equipo Tidymodels\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nAgradecimientos especiales para: Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill y Desirée De Leon" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#varying-the-threshold", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#varying-the-threshold", + "title": "4 - Evaluar modelos", + "section": "Varying the threshold", + "text": "Varying the threshold" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#section-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#section-2", - "title": "1 - Introducción", - "section": "👀", - "text": "👀" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#curvas-roc", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#curvas-roc", + "title": "4 - Evaluar modelos", + "section": "Curvas ROC", + "text": "Curvas ROC\nPara crear una “curva ROC”, osea una curva de característica operativa del receptor hacemos lo siguiente:\n\nCalcular la sensibilidad y especificidad de todos los límites posibles\nGráficar los falsos positivos en el axis X, contra los positivos verdaderos en el axis Y.\n\nYa que la sensibilidad es la proporción de positivos verdaderos, y la especificidad es la de los negativos verdaderos, entonces 1 - especificidad es la proporción de los falsos positivos.\n\nPodemos usar el area debajo de la curva (AUC = area under de curve) ROC como una métrica de clasificación:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-planeamos-hacer-en-este-taller", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-planeamos-hacer-en-este-taller", - "title": "1 - Introducción", - "section": "Que planeamos hacer en este taller", - "text": "Que planeamos hacer en este taller\n\nTu presupuesto de datos\nLas partes de un modelo\nEvaluar modelos\nAfinar modelos\n\n\nThis workshop will well-prepare folks going on to the Advanced tidymodels workshop, which will cover feature engineering and much more on hyperparameter tuning." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#curvas-roc-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#curvas-roc-1", + "title": "4 - Evaluar modelos", + "section": "Curvas ROC ", + "text": "Curvas ROC \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(taxi_ajustado, new_data = taxi_entrenar) %>% \n roc_curve(truth = propina, .pred_si) %>%\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.783 0.209 0.981 \n#> 3 1 1 0.00135\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>% \n roc_auc(truth = propina, .pred_si)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.691" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#section-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#section-3", - "title": "1 - Introducción", - "section": "", - "text": "Salude a sus vecinos 👋" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#gráfica-de-curvas-roc", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#gráfica-de-curvas-roc", + "title": "4 - Evaluar modelos", + "section": "Gráfica de curvas ROC ", + "text": "Gráfica de curvas ROC \n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>% \n roc_curve(truth = propina, .pred_si) %>%\n autoplot()" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático", - "title": "1 - Introducción", - "section": "¿Que es aprendizaje automático?", - "text": "¿Que es aprendizaje automático?" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno", + "title": "4 - Evaluar modelos", + "section": "Tu turno", + "text": "Tu turno\n\nCalcule y gráfique una curva ROC con su modelo\n¿Cuales son los datos que se utilizaron para esta curva ROC?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-1", - "title": "1 - Introducción", - "section": "¿Que es aprendizaje automático?", - "text": "¿Que es aprendizaje automático?\n\n\n¿Este es tu sistema para aprendizaje automático?\nSí, le tiramos los datos a este monton de algebra linear, y después tomamos las repuestas que salen\n¿Y si las respuestas están equivocadas?\nLo empezamos a mezclar hasta que algo se ve como correcto\n\n\nhttps://xkcd.com/1838/" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-1", + "title": "4 - Evaluar modelos", + "section": "Los peligros del sobreajustar ⚠️", + "text": "Los peligros del sobreajustar ⚠️" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-2", - "title": "1 - Introducción", - "section": "¿Que es aprendizaje automático?", - "text": "¿Que es aprendizaje automático?\n\n\nhttps://vas3k.com/blog/machine_learning/" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-2", + "title": "4 - Evaluar modelos", + "section": "Los peligros del sobreajustar ⚠️", + "text": "Los peligros del sobreajustar ⚠️" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-3", - "title": "1 - Introducción", - "section": "¿Que es aprendizaje automático?", - "text": "¿Que es aprendizaje automático?\n\n\n\n\n\nflowchart TB\n au[Aprendizaje\\nAutomático\\nClásico]\n sp[Supervisada]\n au--Datos numéricos o categóricos-->sp\n us[No supervisada]\n au--Datos no están clasificados-->us\n cl[Clasificación]\n sp--Predice categoría-->cl\n rs[Regresión]\n sp--Predice numero-->rs\n ag[Agrupación]\n us--Divide por similitudes-->ag\n rd[Reducción\\n de dimensiones]\n us--Busca dependencias\\nescondidas-->rd\n as[Asociación]\n us--Identifica secuencias-->as\n \n\n\n\n\n\n\n\nhttps://vas3k.com/blog/machine_learning/" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-3", + "title": "4 - Evaluar modelos", + "section": "Los peligros del sobreajustar ⚠️ ", + "text": "Los peligros del sobreajustar ⚠️ \n\ntaxi_ajustado %>%\n augment(taxi_entrenar)\n#> # A tibble: 8,000 × 10\n#> .pred_class .pred_si .pred_no propina distancia compania local dia mes \n#> <fct> <dbl> <dbl> <fct> <dbl> <fct> <fct> <fct> <fct>\n#> 1 si 0.967 0.0333 si 17.2 Chicago In… no Jue Feb \n#> 2 si 0.935 0.0646 si 0.88 City Servi… si Jue Mar \n#> 3 si 0.967 0.0333 si 18.1 otra no Lun Feb \n#> 4 si 0.949 0.0507 si 12.2 Chicago In… no Dom Mar \n#> 5 si 0.821 0.179 si 0.94 Sun Taxi si Sab Abr \n#> 6 si 0.967 0.0333 si 17.5 Flash Cab no Vie Mar \n#> 7 si 0.967 0.0333 si 17.7 otra no Dom Ene \n#> 8 si 0.938 0.0616 si 1.85 Taxicab In… no Vie Abr \n#> 9 si 0.938 0.0616 si 0.53 Sun Taxi no Mar Mar \n#> 10 si 0.931 0.0694 si 6.65 Taxicab In… no Dom Abr \n#> # ℹ 7,990 more rows\n#> # ℹ 1 more variable: hora <int>\n\nA esto le llamamos “resubstitución” ó “repredecir en los datos de entrenamiento”" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#tu-turno", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#tu-turno", - "title": "1 - Introducción", - "section": "Tu turno", - "text": "Tu turno\n\n\n¿Como se relacionan las estadísticas y el aprendizaje automático?\n¿Como se parecen? ¿Cuales son sus diferencias?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-4", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-4", + "title": "4 - Evaluar modelos", + "section": "Los peligros del sobreajustar ⚠️ ", + "text": "Los peligros del sobreajustar ⚠️ \n\ntaxi_ajustado %>%\n augment(taxi_entrenar) %>%\n accuracy(propina, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n\nA esto le llamamos “resubstitución de la estimación”" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-tidymodels", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-tidymodels", - "title": "1 - Introducción", - "section": "¿Que es Tidymodels? ", - "text": "¿Que es Tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.1.1 ──\n#> ✔ broom 1.0.5 ✔ rsample 1.2.0 \n#> ✔ dials 1.2.1 ✔ tibble 3.2.1 \n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1 \n#> ✔ infer 1.0.6 ✔ tune 1.1.2 \n#> ✔ modeldata 1.3.0 ✔ workflows 1.1.4 \n#> ✔ parsnip 1.2.0 ✔ workflowsets 1.0.1 \n#> ✔ purrr 1.0.2 ✔ yardstick 1.3.0 \n#> ✔ recipes 1.0.10\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Learn how to get started at https://www.tidymodels.org/start/" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-5", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-5", + "title": "4 - Evaluar modelos", + "section": "Los peligros del sobreajustar ⚠️ ", + "text": "Los peligros del sobreajustar ⚠️ \n\n\n\ntaxi_ajustado %>%\n augment(taxi_entrenar) %>%\n accuracy(propina, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller", - "title": "1 - Introducción", - "section": "Expectativas del taller", - "text": "Expectativas del taller\n\nEl “mapa” de hoy\nProcesos básicos de los modelos predictivos\nVer la ingenieria de caraterísticas (feature engineering) y afinamiento como extenciones cíclicas" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-6", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-6", + "title": "4 - Evaluar modelos", + "section": "Los peligros del sobreajustar ⚠️ ", + "text": "Los peligros del sobreajustar ⚠️ \n\n\n\ntaxi_ajustado %>%\n augment(taxi_entrenar) %>%\n accuracy(propina, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n\n\n\ntaxi_ajustado %>%\n augment(taxi_prueba) %>%\n accuracy(propina, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.908\n\n\n\n⚠️ Acuerdate que estamos demonstrando el sobreajuste\n\n\n⚠️ No utilizes el set the prueba sino hasta el fin de tu analisis" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-1", - "title": "1 - Introducción", - "section": "Expectativas del taller", - "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#ja-ja-ja-estoy-en-peligro", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#ja-ja-ja-estoy-en-peligro", + "title": "4 - Evaluar modelos", + "section": "“Ja-ja-ja, estoy en peligro”", + "text": "“Ja-ja-ja, estoy en peligro”" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-2", - "title": "1 - Introducción", - "section": "Expectativas del taller", - "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n tr --> dt" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-1", + "title": "4 - Evaluar modelos", + "section": "Tu turno", + "text": "Tu turno\n\nUsa augment() y una función de métrica para calcular una métrica de classificación, por ejemplo brier_class()\nCalcula las métricas para los datos de entrenamiento y de prueba para demonstrar el sobreajuste\nNota la evidencia de sobreajuste ⚠️\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-3", - "title": "1 - Introducción", - "section": "Expectativas del taller", - "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n tr --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n tr --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n tr --> rf" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-7", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-7", + "title": "4 - Evaluar modelos", + "section": "Los peligros del sobreajustar ⚠️ ", + "text": "Los peligros del sobreajustar ⚠️ \n\n\n\ntaxi_ajustado %>%\n augment(taxi_entrenar) %>%\n brier_class(propina, .pred_si)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0632\n\n\n\ntaxi_ajustado %>%\n augment(taxi_prueba) %>%\n brier_class(propina, .pred_si)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0782\n\n\n\n¿Que tal si queremos comparar más modelos?\n\n\n…y comparar configuraciones para los modelos?\n\n\nY tambien queremos saber si las diferencias son importantes" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-4", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-4", - "title": "1 - Introducción", - "section": "Expectativas del taller", - "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#remuestreo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#remuestreo", + "title": "4 - Evaluar modelos", + "section": "Remuestreo", + "text": "Remuestreo\n\n\n\n\n\nflowchart TD\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rm1[Remuestreo 1]\n style rm1 fill:#fff,stroke:#666,color:#000\n tr --> rm1\n rm2[Remuestreo 2]\n style rm2 fill:#fff,stroke:#666,color:#000\n tr --> rm2\n rm3[Remuestreo B]\n style rm3 fill:#fff,stroke:#666,color:#000\n tr --> rm3\n an1[Analysis]\n style an1 fill:#FBE9BF,stroke:#666,color:#000\n rm1 --> an1\n vl1[Validación]\n style vl1 fill:#E5E7FD,stroke:#666,color:#000\n rm1 --> vl1\n an2[Analysis]\n style an2 fill:#FBE9BF,stroke:#666,color:#000\n rm2 --> an2\n vl2[Validación]\n style vl2 fill:#E5E7FD,stroke:#666,color:#000\n rm2 --> vl2 \n an3[Analysis]\n style an3 fill:#FBE9BF,stroke:#666,color:#000\n rm3 --> an3\n vl3[Validación]\n style vl3 fill:#E5E7FD,stroke:#666,color:#000\n rm3 --> vl3" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-5", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-5", - "title": "1 - Introducción", - "section": "Expectativas del taller", - "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada", + "title": "4 - Evaluar modelos", + "section": "Validación cruzada", + "text": "Validación cruzada" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-6", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-6", - "title": "1 - Introducción", - "section": "Expectativas del taller", - "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#FBE9BF,stroke:#666,color:#000\n sm --> fm\n tr --> fm" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-1", + "title": "4 - Evaluar modelos", + "section": "Validación cruzada", + "text": "Validación cruzada" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-7", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-7", - "title": "1 - Introducción", - "section": "Expectativas del taller", - "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#FBE9BF,stroke:#666,color:#000\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#E5E7FD,stroke:#666,color:#000\n fm --> vm\n ts --> vm" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-2", + "title": "4 - Evaluar modelos", + "section": "Tu turno", + "text": "Tu turno\n\nSi usamos 10 plieges (folds), cual es el porcentaje de datos de entrenamiento\n\ncuantos terminan en análisis\ncuantos terminan en evaluación (assesment)\n\n…para cada pliege?\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#instalemos-unos-paquetes", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#instalemos-unos-paquetes", - "title": "1 - Introducción", - "section": "Instalemos unos paquetes", - "text": "Instalemos unos paquetes\n\npkgs <- \n c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\",\n \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\",\n \"splines2\", \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \n \"vetiver\", \"remotes\")\n\ninstall.packages(pkgs)" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-2", + "title": "4 - Evaluar modelos", + "section": "Validación cruzada ", + "text": "Validación cruzada \n\nvfold_cv(taxi_entrenar) \n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#nuestras-versiones", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#nuestras-versiones", - "title": "1 - Introducción", - "section": "Nuestras versiones", - "text": "Nuestras versiones\nR version 4.3.2 (2023-10-31), Quarto (1.4.550)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.2.1\n\n\nbroom\n1.0.5\n\n\ndials\n1.2.1\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nembed\n1.1.3\n\n\nfinetune\n1.1.0\n\n\nggplot2\n3.5.0\n\n\nlightgbm\n4.3.0\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nlme4\n1.1-35.1\n\n\nmodeldata\n1.3.0\n\n\nparsnip\n1.2.0\n\n\nplumber\n1.2.1\n\n\nprobably\n1.0.3\n\n\npurrr\n1.0.2\n\n\nranger\n0.16.0\n\n\nrecipes\n1.0.10\n\n\nremotes\n2.4.2.1\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nrpart\n4.1.23\n\n\nrpart.plot\n3.1.2\n\n\nrsample\n1.2.0\n\n\nrules\n1.0.2\n\n\nscales\n1.3.0\n\n\nsplines2\n0.5.1\n\n\nstacks\n1.0.3\n\n\ntext2vec\n0.6.4\n\n\ntextrecipes\n1.0.6\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.1.1\n\n\ntidyr\n1.3.1\n\n\ntune\n1.1.2\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.1.4\n\n\nworkflowsets\n1.0.1\n\n\nyardstick\n1.3.0" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-3", + "title": "4 - Evaluar modelos", + "section": "Validación cruzada ", + "text": "Validación cruzada \n¿Que hay en este?\n\ntaxi_plieges <- vfold_cv(taxi_entrenar)\ntaxi_plieges$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n\n\nTalk about a list column, storing non-atomic types in dataframe" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#the-raw-taxi-data-set", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#the-raw-taxi-data-set", - "title": "Extras - Time-based data splitting", - "section": "The raw taxi data set", - "text": "The raw taxi data set\nWe prepared the data set specifically for this introductory workshop.\nIt looked similar to this:\n\nglimpse(taxi_raw)\n#> Rows: 10,000\n#> Columns: 24\n#> $ trip_id <chr> \"3ac8d4412642a35e9b9a493285814d7983d5a159\",…\n#> $ taxi_id <chr> \"391317d70c5d06deec744062c4595dc1958b200fda…\n#> $ trip_start_timestamp <dttm> 2023-06-10 18:45:00, 2023-05-21 21:30:00, …\n#> $ trip_end_timestamp <dttm> 2023-06-10 19:30:00, 2023-05-21 21:45:00, …\n#> $ trip_seconds <dbl> 3258, 839, 476, 2220, 1588, 2270, 1575, 267…\n#> $ trip_miles <dbl> 17.02, 2.16, 1.05, 17.40, 17.62, 16.36, 18.…\n#> $ pickup_census_tract <dbl> 17031980000, 17031839100, 17031320100, 1703…\n#> $ dropoff_census_tract <dbl> 17031081403, 17031081300, 17031081403, 1703…\n#> $ pickup_community_area <dbl> 76, 32, 32, 76, 32, 76, 76, 32, 32, 33, 8, …\n#> $ dropoff_community_area <dbl> 8, 8, 8, 32, 76, 8, 32, 76, 28, 32, 7, 33, …\n#> $ fare <dbl> 44.50, 10.00, 6.75, 45.00, 44.25, 41.25, 44…\n#> $ tips <dbl> 12.25, 4.00, 2.00, 9.90, 8.00, 9.15, 12.31,…\n#> $ tolls <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ extras <dbl> 4.0, 1.0, 0.0, 4.0, 2.0, 4.0, 4.0, 0.0, 0.0…\n#> $ trip_total <dbl> 61.25, 15.50, 9.25, 58.90, 54.75, 54.90, 61…\n#> $ payment_type <chr> \"Credit Card\", \"Credit Card\", \"Credit Card\"…\n#> $ company <chr> \"Taxicab Insurance Agency Llc\", \"Chicago In…\n#> $ pickup_centroid_latitude <dbl> 41.97907, 41.88099, 41.88499, 41.97907, 41.…\n#> $ pickup_centroid_longitude <dbl> -87.90304, -87.63275, -87.62099, -87.90304,…\n#> $ pickup_centroid_location <chr> \"POINT (-87.9030396611 41.9790708201)\", \"PO…\n#> $ dropoff_centroid_latitude <dbl> 41.89092, 41.89833, 41.89092, 41.87102, 41.…\n#> $ dropoff_centroid_longitude <dbl> -87.61887, -87.62076, -87.61887, -87.63141,…\n#> $ dropoff_centroid_location <chr> \"POINT (-87.6188683546 41.8909220259)\", \"PO…\n#> $ tip <fct> yes, yes, yes, yes, yes, yes, yes, yes, yes…\n\n\ntrip_start_time has date and time rounded to “quarters of the hour”." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-4", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-4", + "title": "4 - Evaluar modelos", + "section": "Validación cruzada ", + "text": "Validación cruzada \n\nvfold_cv(taxi_entrenar, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [6400/1600]> Fold1\n#> 2 <split [6400/1600]> Fold2\n#> 3 <split [6400/1600]> Fold3\n#> 4 <split [6400/1600]> Fold4\n#> 5 <split [6400/1600]> Fold5" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#time-nature-of-the-data", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#time-nature-of-the-data", - "title": "Extras - Time-based data splitting", - "section": "Time nature of the data", - "text": "Time nature of the data\nWe assumed only the month, day of the week, and hour mattered and treated each observation as independent.\n\nIf the data have a strong time component, all your data splitting strategies should support the model in estimating temporal trends.\n\n\nThus, don’t sample randomly because this breaks up the time component!" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-5", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-5", + "title": "4 - Evaluar modelos", + "section": "Validación cruzada ", + "text": "Validación cruzada \n\nvfold_cv(taxi_entrenar, strata = propina)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10\n\n\nEstratificar usualmente ayuda, y con pocos malas consecuencias" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#splitting-with-time-component", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#splitting-with-time-component", - "title": "Extras - Time-based data splitting", - "section": "Splitting with time component ", - "text": "Splitting with time component \nThe more recent observations are assumed to be more similar to new data, so initial_time_split() puts them into the test set.\nThe function assumes that the data are already ordered.\n\ntaxi_raw <- taxi_raw %>%\n arrange(trip_start_timestamp)\n\ntaxi_split <- initial_time_split(taxi_raw, prop = 3 / 4)\ntaxi_split\n#> <Training/Testing/Total>\n#> <7500/2500/10000>\n\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)\n\nnrow(taxi_train)\n#> [1] 7500\n \nnrow(taxi_test)\n#> [1] 2500" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-6", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-6", + "title": "4 - Evaluar modelos", + "section": "Validación cruzada ", + "text": "Validación cruzada \nUsaremos esto:\n\nset.seed(123)\ntaxi_plieges <- vfold_cv(taxi_entrenar, v = 10, strata = propina)\ntaxi_plieges\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10\n\n\nEspecifica la semilla (seed) cuando estas creando remuestreos" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#time-series-resampling", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#time-series-resampling", - "title": "Extras - Time-based data splitting", - "section": "Time series resampling", - "text": "Time series resampling\nThe same idea also applies to resampling: the newer observations go into the assessment set.\nFor example:\n\nFold 1: Take the first X weeks of data as the analysis set, and the next 3 weeks as the assessment set.\nFold 2: Take weeks 2 to X + 1 as the analysis set, and the next 3 weeks as the assessment set.\nand so on" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#ajustemos-nuestro-modelo-usando-los-remuestreos", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#ajustemos-nuestro-modelo-usando-los-remuestreos", + "title": "4 - Evaluar modelos", + "section": "Ajustemos nuestro modelo usando los remuestreos", + "text": "Ajustemos nuestro modelo usando los remuestreos\n\ntaxi_res <- fit_resamples(arbol_flujo, taxi_plieges)\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]>" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#rolling-origin-forecast-resampling", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#rolling-origin-forecast-resampling", - "title": "Extras - Time-based data splitting", - "section": "Rolling origin forecast resampling", - "text": "Rolling origin forecast resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done either way." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo", + "title": "4 - Evaluar modelos", + "section": "Evaluando la calidad del modelo ", + "text": "Evaluando la calidad del modelo \n\ntaxi_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.915 10 0.00309 Preprocessor1_Model1\n#> 2 roc_auc binary 0.624 10 0.0105 Preprocessor1_Model1\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nPodemos medir correctamente la la calidad del modelo usando solo los datos de entrenamiento 🎉" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling", - "title": "Extras - Time-based data splitting", - "section": "Times series resampling ", - "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n\n\n\n\n )\n\nUse the trip_start_timestamp column to find the date data." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#comparando-las-métricas", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#comparando-las-métricas", + "title": "4 - Evaluar modelos", + "section": "Comparando las métricas ", + "text": "Comparando las métricas \n¿Que diferencia hay entre las métricas usando los datos de remuestreo, y usando los datos de entrenamiento y prueba?\n\n\n\ntaxi_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.915 10\n#> 2 roc_auc 0.624 10\n\n\nEl ROC AUC antes era:\n\n0.69 para el set the entrenamiento\n0.64 para el set the prueba\n\n\n\nAcuerdate que:\n⚠️ Los datos de entrenamiento da métricas demasiado optimísticas\n⚠️ Los datos de prueba son valiosos" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-1", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-1", - "title": "Extras - Time-based data splitting", - "section": "Times series resampling ", - "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n period = \"week\",\n\n\n\n )\n\nOur units will be in weeks." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-1", + "title": "4 - Evaluar modelos", + "section": "Evaluando la calidad del modelo ", + "text": "Evaluando la calidad del modelo \n\n\nctrl_taxi <- control_resamples(save_pred = TRUE)\ntaxi_res <- fit_resamples(arbol_flujo, taxi_plieges, control = ctrl_taxi)\n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-2", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-2", - "title": "Extras - Time-based data splitting", - "section": "Times series resampling ", - "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n period = \"week\",\n lookback = 8\n \n \n )\n\nEvery analysis set has 8 weeks of data." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-2", + "title": "4 - Evaluar modelos", + "section": "Evaluando la calidad del modelo ", + "text": "Evaluando la calidad del modelo \n\n# Guarde los resultados de las evaluaciones\ntaxi_preds <- collect_predictions(taxi_res)\ntaxi_preds\n#> # A tibble: 8,000 × 7\n#> id .pred_si .pred_no .row .pred_class propina .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 Fold01 0.938 0.0615 14 si si Preprocessor1_Model1\n#> 2 Fold01 0.946 0.0544 19 si si Preprocessor1_Model1\n#> 3 Fold01 0.973 0.0269 33 si si Preprocessor1_Model1\n#> 4 Fold01 0.903 0.0971 43 si si Preprocessor1_Model1\n#> 5 Fold01 0.973 0.0269 74 si si Preprocessor1_Model1\n#> 6 Fold01 0.903 0.0971 103 si si Preprocessor1_Model1\n#> 7 Fold01 0.915 0.0851 104 si no Preprocessor1_Model1\n#> 8 Fold01 0.903 0.0971 124 si si Preprocessor1_Model1\n#> 9 Fold01 0.667 0.333 126 si si Preprocessor1_Model1\n#> 10 Fold01 0.949 0.0510 128 si si Preprocessor1_Model1\n#> # ℹ 7,990 more rows" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-3", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-3", - "title": "Extras - Time-based data splitting", - "section": "Times series resampling ", - "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n period = \"week\",\n lookback = 8,\n assess_stop = 3,\n\n )\n\nEvery assessment set has 3 weeks of data." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-3", + "title": "4 - Evaluar modelos", + "section": "Evaluando la calidad del modelo ", + "text": "Evaluando la calidad del modelo \n\ntaxi_preds %>% \n group_by(id) %>%\n taxi_metrics(truth = propina, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.905\n#> 2 Fold02 accuracy binary 0.925\n#> 3 Fold03 accuracy binary 0.926\n#> 4 Fold04 accuracy binary 0.915\n#> 5 Fold05 accuracy binary 0.902\n#> 6 Fold06 accuracy binary 0.912\n#> 7 Fold07 accuracy binary 0.906\n#> 8 Fold08 accuracy binary 0.91 \n#> 9 Fold09 accuracy binary 0.918\n#> 10 Fold10 accuracy binary 0.931\n#> # ℹ 20 more rows" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-4", - "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-4", - "title": "Extras - Time-based data splitting", - "section": "Times series resampling ", - "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n period = \"week\",\n lookback = 8,\n assess_stop = 3,\n step = 1\n )\n\nIncrement by 1 week\n\n\ntaxi_rs$splits[[1]] %>% assessment() %>% pluck(\"trip_start_timestamp\") %>% range()\n#> [1] \"2023-03-02 05:15:00 UTC\" \"2023-03-22 22:00:00 UTC\"\n\ntaxi_rs$splits[[2]] %>% assessment() %>% pluck(\"trip_start_timestamp\") %>% range()\n#> [1] \"2023-03-09 07:00:00 UTC\" \"2023-03-29 21:15:00 UTC\"\n\ntaxi_rs$splits[[3]] %>% assessment() %>% pluck(\"trip_start_timestamp\") %>% range()\n#> [1] \"2023-03-16 06:30:00 UTC\" \"2023-04-05 23:45:00 UTC\"" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#donde-estan-los-modelos-que-ajustamos", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#donde-estan-los-modelos-que-ajustamos", + "title": "4 - Evaluar modelos", + "section": "¿Donde estan los modelos que ajustamos? ", + "text": "¿Donde estan los modelos que ajustamos? \n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" }, { - "objectID": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#your-turn", - "href": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#your-turn", - "title": "6 - Wrapping up", - "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bootstrapping", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bootstrapping", + "title": "4 - Evaluar modelos", + "section": "Bootstrapping", + "text": "Bootstrapping" }, { - "objectID": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#resources-to-keep-learning", - "href": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#resources-to-keep-learning", - "title": "6 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bootstrapping-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bootstrapping-1", + "title": "4 - Evaluar modelos", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(taxi_entrenar)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [8000/2902]> Bootstrap01\n#> 2 <split [8000/2916]> Bootstrap02\n#> 3 <split [8000/3004]> Bootstrap03\n#> 4 <split [8000/2979]> Bootstrap04\n#> 5 <split [8000/2961]> Bootstrap05\n#> 6 <split [8000/2962]> Bootstrap06\n#> 7 <split [8000/3026]> Bootstrap07\n#> 8 <split [8000/2926]> Bootstrap08\n#> 9 <split [8000/2972]> Bootstrap09\n#> 10 <split [8000/2972]> Bootstrap10\n#> # ℹ 15 more rows" }, { - "objectID": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#feedback", - "href": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#feedback", - "title": "6 - Wrapping up", - "section": "Feedback", - "text": "Feedback\n\nPlease go to http://pos.it/conf-workshop-survey. Your feedback is crucial!\nData from the survey informs curriculum and format decisions for future conf workshops, and we really appreciate you taking the time to provide it." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---donde-estamos", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---donde-estamos", + "title": "4 - Evaluar modelos", + "section": "Expectativas del taller - Donde estamos", + "text": "Expectativas del taller - Donde estamos\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestras]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#fff,stroke:#eee,color:#ddd\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#fff,stroke:#eee,color:#ddd\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#fff,stroke:#eee,color:#ddd\n fm --> vm\n ts --> vm" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#looking-at-predictions", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#looking-at-predictions", - "title": "4 - Evaluating models", - "section": "Looking at predictions", - "text": "Looking at predictions\n\naugment(taxi_fit, new_data = taxi_train) %>%\n relocate(tip, .pred_class, .pred_yes, .pred_no)\n#> # A tibble: 8,000 × 10\n#> tip .pred_class .pred_yes .pred_no distance company local dow month hour\n#> <fct> <fct> <dbl> <dbl> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes yes 0.967 0.0333 17.2 Chicag… no Thu Feb 16\n#> 2 yes yes 0.935 0.0646 0.88 City S… yes Thu Mar 8\n#> 3 yes yes 0.967 0.0333 18.1 other no Mon Feb 18\n#> 4 yes yes 0.949 0.0507 12.2 Chicag… no Sun Mar 21\n#> 5 yes yes 0.821 0.179 0.94 Sun Ta… yes Sat Apr 23\n#> 6 yes yes 0.967 0.0333 17.5 Flash … no Fri Mar 12\n#> 7 yes yes 0.967 0.0333 17.7 other no Sun Jan 6\n#> 8 yes yes 0.938 0.0616 1.85 Taxica… no Fri Apr 12\n#> 9 yes yes 0.938 0.0616 0.53 Sun Ta… no Tue Mar 18\n#> 10 yes yes 0.931 0.0694 6.65 Taxica… no Sun Apr 11\n#> # ℹ 7,990 more rows" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-3", + "title": "4 - Evaluar modelos", + "section": "Tu turno", + "text": "Tu turno\n\nCrea un:\n\nSet de validación cruzada tipo Monte Carlo\nSet de validación\n\nNo te olvides de usar set.seed()\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-tipo-monte-carlo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-tipo-monte-carlo", + "title": "4 - Evaluar modelos", + "section": "Validación cruzada tipo Monte Carlo ", + "text": "Validación cruzada tipo Monte Carlo \n\nset.seed(322)\nmc_cv(taxi_entrenar, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6000/2000]> Resample01\n#> 2 <split [6000/2000]> Resample02\n#> 3 <split [6000/2000]> Resample03\n#> 4 <split [6000/2000]> Resample04\n#> 5 <split [6000/2000]> Resample05\n#> 6 <split [6000/2000]> Resample06\n#> 7 <split [6000/2000]> Resample07\n#> 8 <split [6000/2000]> Resample08\n#> 9 <split [6000/2000]> Resample09\n#> 10 <split [6000/2000]> Resample10" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix-1", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(taxi_fit, new_data = taxi_train) %>%\n conf_mat(truth = tip, estimate = .pred_class)\n#> Truth\n#> Prediction yes no\n#> yes 7341 536\n#> no 43 80" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#set-de-validación", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#set-de-validación", + "title": "4 - Evaluar modelos", + "section": "Set de validación ", + "text": "Set de validación \n\nset.seed(853)\ntaxi_val_split <- initial_validation_split(taxi, strata = propina)\nvalidation_set(taxi_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6000/2000]> validation\n\n\nUn set de validación es solamente otro tipo de remuestreo" }, - { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix-2", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix-2", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(taxi_fit, new_data = taxi_train) %>%\n conf_mat(truth = tip, estimate = .pred_class) %>%\n autoplot(type = \"heatmap\")" + { + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bosque-aleatorio-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bosque-aleatorio-1", + "title": "4 - Evaluar modelos", + "section": "Bosque aleatorio 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Bosque aleatorio 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsambla varios árboles de decisión\n¡Todos los árboles votan! 🗳️\nAgregación tipo bootstrap + muestreo aleatorio del predictor\n\n\n\nUsualmente funciona bien sin usar afinamiento, pero solo mientras hayan suficientes árboles" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n accuracy(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#crear-un-modelo-de-bosque-aleatorio", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#crear-un-modelo-de-bosque-aleatorio", + "title": "4 - Evaluar modelos", + "section": "Crear un modelo de bosque aleatorio ", + "text": "Crear un modelo de bosque aleatorio \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-accuracy", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-accuracy", - "title": "4 - Evaluating models", - "section": "Dangers of accuracy ", - "text": "Dangers of accuracy \nWe need to be careful of using accuracy() since it can give “good” performance by only predicting one way with imbalanced data\n\naugment(taxi_fit, new_data = taxi_train) %>%\n mutate(.pred_class = factor(\"yes\", levels = c(\"yes\", \"no\"))) %>%\n accuracy(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.923" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#crear-un-modelo-de-bosque-aleatorio-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#crear-un-modelo-de-bosque-aleatorio-1", + "title": "4 - Evaluar modelos", + "section": "Crear un modelo de bosque aleatorio ", + "text": "Crear un modelo de bosque aleatorio \n\nrf_wflow <- workflow(propina ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-1", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n sensitivity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.994" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-4", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-4", + "title": "4 - Evaluar modelos", + "section": "Tu turno", + "text": "Tu turno\n\nUsa fit_resamples() y rf_wflow para lo siguiente:\n\nQuedarse con las predicciones\nCalcular las métricas\n\n\n\n\n−+\n08:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-2", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-2", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n sensitivity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.994\n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n specificity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.130" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-4", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-4", + "title": "4 - Evaluar modelos", + "section": "Evaluando la calidad del modelo ", + "text": "Evaluando la calidad del modelo \n\nctrl_taxi <- control_resamples(save_pred = TRUE)\n\n# Bosque aleatorio usan numeros al azar, asi que asegurate de definir la semilla\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, taxi_plieges, control = ctrl_taxi)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.924 10 0.00323 Preprocessor1_Model1\n#> 2 roc_auc binary 0.611 10 0.0149 Preprocessor1_Model1" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-3", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-3", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_fit, new_data = taxi_train) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n#> 2 specificity binary 0.130\n#> 3 sensitivity binary 0.994" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---donde-estamos-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---donde-estamos-1", + "title": "4 - Evaluar modelos", + "section": "Expectativas del taller - Donde estamos", + "text": "Expectativas del taller - Donde estamos\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestras]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#fff,stroke:#eee,color:#ddd\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#fff,stroke:#eee,color:#ddd\n fm --> vm\n ts --> vm" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-4", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-4", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_fit, new_data = taxi_train) %>%\n group_by(local) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 6 × 4\n#> local .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 yes accuracy binary 0.898\n#> 2 no accuracy binary 0.935\n#> 3 yes specificity binary 0.169\n#> 4 no specificity binary 0.116\n#> 5 yes sensitivity binary 0.987\n#> 6 no sensitivity binary 0.996" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#el-ajuste-final", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#el-ajuste-final", + "title": "4 - Evaluar modelos", + "section": "El ajuste final ", + "text": "El ajuste final \nDigamos que estamos satisfechos con usar nuestro modelo de bosque aleatorio\nAjustemos el modelo usando todos los datos en el set the entrenamiento, y despues midamos la calidad del modelo con el set the prueba\n\nHemos usado fit() y predict() (+ augment()), pero hay un atajo:\n\n# taxi_separar has train + test info\najuste_final <- last_fit(rf_wflow, taxi_separar) \n\najuste_final\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [8000/2000]> train/test split <tibble> <tibble> <tibble> <workflow>" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#two-class-data", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#two-class-data", - "title": "4 - Evaluating models", - "section": "Two class data", - "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final", + "title": "4 - Evaluar modelos", + "section": "¿Que contiene ajuste_final? ", + "text": "¿Que contiene ajuste_final? \n\ncollect_metrics(ajuste_final)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.914 Preprocessor1_Model1\n#> 2 roc_auc binary 0.630 Preprocessor1_Model1\n\n\nLas métricas fueron calculadas con el set de prueba" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#varying-the-threshold", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#varying-the-threshold", - "title": "4 - Evaluating models", - "section": "Varying the threshold", - "text": "Varying the threshold" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final-1", + "title": "4 - Evaluar modelos", + "section": "¿Que contiene ajuste_final? ", + "text": "¿Que contiene ajuste_final? \n\ncollect_predictions(ajuste_final)\n#> # A tibble: 2,000 × 7\n#> id .pred_si .pred_no .row .pred_class propina .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 train/test split 0.949 0.0513 4 si si Preprocessor1_M…\n#> 2 train/test split 0.922 0.0777 10 si si Preprocessor1_M…\n#> 3 train/test split 0.961 0.0395 19 si si Preprocessor1_M…\n#> 4 train/test split 0.889 0.111 23 si si Preprocessor1_M…\n#> 5 train/test split 0.936 0.0639 28 si si Preprocessor1_M…\n#> 6 train/test split 0.969 0.0306 34 si si Preprocessor1_M…\n#> 7 train/test split 0.962 0.0377 35 si si Preprocessor1_M…\n#> 8 train/test split 0.939 0.0606 38 si si Preprocessor1_M…\n#> 9 train/test split 0.986 0.0138 40 si si Preprocessor1_M…\n#> 10 train/test split 0.957 0.0431 42 si no Preprocessor1_M…\n#> # ℹ 1,990 more rows" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curves", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curves", - "title": "4 - Evaluating models", - "section": "ROC curves", - "text": "ROC curves\nTo make an ROC (receiver operator characteristic) curve, we:\n\ncalculate the sensitivity and specificity for all possible thresholds\nplot false positive rate (x-axis) versus true positive rate (y-axis)\n\ngiven that sensitivity is the true positive rate, and specificity is the true negative rate. Hence 1 - specificity is the false positive rate.\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final-2", + "title": "4 - Evaluar modelos", + "section": "¿Que contiene ajuste_final? ", + "text": "¿Que contiene ajuste_final? \n\nextract_workflow(ajuste_final)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 8000 \n#> Number of independent variables: 6 \n#> Mtry: 2 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.07070342\n\n\nUse this for prediction on new data, like for deploying" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curves-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curves-1", - "title": "4 - Evaluating models", - "section": "ROC curves ", - "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_curve(truth = tip, .pred_yes) %>%\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.783 0.209 0.981 \n#> 3 1 1 0.00135\n\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_auc(truth = tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.691" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---ya-llegamos", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---ya-llegamos", + "title": "4 - Evaluar modelos", + "section": "Expectativas del taller - ¡Ya llegamos!", + "text": "Expectativas del taller - ¡Ya llegamos!\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#FBE9BF,stroke:#666,color:#000\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#E5E7FD,stroke:#666,color:#000\n fm --> vm\n ts --> vm" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curve-plot", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curve-plot", - "title": "4 - Evaluating models", - "section": "ROC curve plot ", - "text": "ROC curve plot \n\n\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_curve(truth = tip, .pred_yes) %>%\n autoplot()" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section", + "title": "2 - Tu presupuesto de datos", + "section": "", + "text": "“¡¡Que emoción!!”" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago", + "title": "2 - Tu presupuesto de datos", + "section": "Datos de viajes en taxi de Chicago", + "text": "Datos de viajes en taxi de Chicago\n\n\n\nLa ciudad de Chicago publica datos a nivel de viaje\nSacamos una muestra de 10,000 viajes que ocurrieron al principio del 2022\n\n\n\n\n\nhttps://www.svgrepo.com/svg/8322/taxi" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-1", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago-1", + "title": "2 - Tu presupuesto de datos", + "section": "Datos de viajes en taxi de Chicago", + "text": "Datos de viajes en taxi de Chicago\n\n\n\nN = 10,000\nUn resultado nominale, propina, con los niveles \"si\" y \"no\"\nVarias variables nominales, por ejemplo la identificación del Taxi, y el tipo de pago\nVarias variables numéricas, por ejemplo la distancia del viaje, y los subtotales del cargo por el viaje\n\n\n\n\n\nCredit: https://unsplash.com/photos/7_r85l4eht8\n\n\n“Fare subtotals” refers to the fare itself, tax, tolls, tip amount.\nActual variables in our data:\ntip: Whether the rider left a tip. A factor with levels “yes” and “no”.\ndistance: The trip distance, in odometer miles.\ncompany: The taxi company, as a factor. Companies that occurred few times were binned as “other”.\nlocal: Whether the trip started in the same community area as it began. See the source data for community area values.\ndow: The day of the week in which the trip began, as a factor.\nmonth: The month in which the trip began, as a factor.\nhour: The hour of the day in which the trip began, as a numeric." }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-2", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-2", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#se-pueden-usar-estas-variables", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#se-pueden-usar-estas-variables", + "title": "2 - Tu presupuesto de datos", + "section": "¿Se pueden usar estas variables?", + "text": "¿Se pueden usar estas variables?\n\n¿Será ético, o hasta legal, utilizar esta variable?\n¿La variable estará disponible al momento de predecir?\n¿La variable contribuye a la explicación de los resultados?" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-3", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-3", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\ntaxi_fit %>%\n augment(taxi_train)\n#> # A tibble: 8,000 × 10\n#> tip distance company local dow month hour .pred_class .pred_yes .pred_no\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int> <fct> <dbl> <dbl>\n#> 1 yes 17.2 Chicag… no Thu Feb 16 yes 0.967 0.0333\n#> 2 yes 0.88 City S… yes Thu Mar 8 yes 0.935 0.0646\n#> 3 yes 18.1 other no Mon Feb 18 yes 0.967 0.0333\n#> 4 yes 12.2 Chicag… no Sun Mar 21 yes 0.949 0.0507\n#> 5 yes 0.94 Sun Ta… yes Sat Apr 23 yes 0.821 0.179 \n#> 6 yes 17.5 Flash … no Fri Mar 12 yes 0.967 0.0333\n#> 7 yes 17.7 other no Sun Jan 6 yes 0.967 0.0333\n#> 8 yes 1.85 Taxica… no Fri Apr 12 yes 0.938 0.0616\n#> 9 yes 0.53 Sun Ta… no Tue Mar 18 yes 0.938 0.0616\n#> 10 yes 6.65 Taxica… no Sun Apr 11 yes 0.931 0.0694\n#> # ℹ 7,990 more rows\n\nWe call this “resubstitution” or “repredicting the training set”" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago-2", + "title": "2 - Tu presupuesto de datos", + "section": "Datos de viajes en taxi de Chicago", + "text": "Datos de viajes en taxi de Chicago\nBaja los datos, y cargarlos en tu sesión de R\n\ndownload.file(\"https://github.com/edgararuiz/tidymodels-workshops/blob/main/archive/2024-03-conectaR-spanish/taxi.rds\",\n \"taxi.rds\")\n\ntaxi <- readRDS(taxi.rds)\n\n\ntaxi\n#> # A tibble: 10,000 × 7\n#> propina distancia compania local dia mes hora\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 si 17.2 Chicago Independents no Jue Feb 16\n#> 2 si 0.88 City Service si Jue Mar 8\n#> 3 si 18.1 otra no Lun Feb 18\n#> 4 si 20.7 Chicago Independents no Lun Abr 8\n#> 5 si 12.2 Chicago Independents no Dom Mar 21\n#> 6 si 0.94 Sun Taxi si Sab Abr 23\n#> 7 si 17.5 Flash Cab no Vie Mar 12\n#> 8 si 17.7 otra no Dom Ene 6\n#> 9 si 1.85 Taxicab Insurance Agency Llc no Vie Abr 12\n#> 10 si 1.47 City Service no Mar Mar 14\n#> # ℹ 9,990 more rows" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-4", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-4", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n\nWe call this a “resubstitution estimate”" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos", + "title": "2 - Tu presupuesto de datos", + "section": "Separando y “gastando” datos", + "text": "Separando y “gastando” datos\nPara el aprendizaje automatico, separamos los datos, unos para entranamiento, y los otros para la “prueba final”:\n\n\nLos datos de entrenamiento se utilizan para estimar los parametros del modelo\nLos datos de prueba se ponen aparte para medir la calidad del modelo\n\n\n\n🚫 No utilize los datos de prueba durante el entrenamiento 🚫" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-5", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-5", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-1", + "title": "2 - Tu presupuesto de datos", + "section": "Separando y “gastando” datos", + "text": "Separando y “gastando” datos" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-6", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-6", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n\n\n\ntaxi_fit %>%\n augment(taxi_test) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.908\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-2", + "title": "2 - Tu presupuesto de datos", + "section": "Separando y “gastando” datos", + "text": "Separando y “gastando” datos\n\nGastando mucho datos en el entrenamiento previene que tengamos una buena manera de medir la calidad del modelo\n\n\n\nGastando mucho datos en prueba previente que tegamos una buena estimación de los parametros" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-1", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse augment() and a metric function to compute a classification metric like brier_class().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno", + "title": "2 - Tu presupuesto de datos", + "section": "Tu turno", + "text": "Tu turno\n\n¿Cuando deberiamos separar nuestros datos?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-7", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-7", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n brier_class(tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0632\n\n\n\ntaxi_fit %>%\n augment(taxi_test) %>%\n brier_class(tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0782\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#la-separación-inicial", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#la-separación-inicial", + "title": "2 - Tu presupuesto de datos", + "section": "La separación inicial ", + "text": "La separación inicial \n\nset.seed(123)\ntaxi_separar <- initial_split(taxi)\ntaxi_separar\n#> <Training/Testing/Total>\n#> <7500/2500/10000>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#para-que-es-set.seed", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#para-que-es-set.seed", + "title": "2 - Tu presupuesto de datos", + "section": "¿Para que es set.seed()?", + "text": "¿Para que es set.seed()?\n\nPara el propósito de separar datos, R puede generar números “seudo-aleatorios”, que aunque parezcan comportarse como números aleatorios, en realidad se generan de una manera específica basada en una “semilla” (seed)\n\n\nSi utilizamos la misma semilla nos permite reproducir los mismos resultados.\n\n\nEl número en sí que elijes realmente no importa. Lo que si importa, es que no experimentes con diferentes números de la semilla con el proposito de mejorar la calidad del modelo." }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-1", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#accediendo-los-datos", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#accediendo-los-datos", + "title": "2 - Tu presupuesto de datos", + "section": "Accediendo los datos ", + "text": "Accediendo los datos \n\ntaxi_entrenar <- training(taxi_separar)\ntaxi_test <- testing(taxi_separar)" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-2", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-2", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#el-set-the-entrenamiento", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#el-set-the-entrenamiento", + "title": "2 - Tu presupuesto de datos", + "section": "El set the entrenamiento ", + "text": "El set the entrenamiento \n\ntaxi_entrenar\n#> # A tibble: 7,500 × 7\n#> propina distancia compania local dia mes hora\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 si 0.7 Taxi Affiliation Services si Mar Mar 18\n#> 2 si 0.99 Sun Taxi si Mar Ene 8\n#> 3 si 1.78 otra no Sab Mar 22\n#> 4 si 0 Taxi Affiliation Services si Mie Abr 15\n#> 5 si 0 Taxi Affiliation Services no Dom Ene 21\n#> 6 si 2.3 otra no Sab Abr 21\n#> 7 si 6.35 Sun Taxi no Mie Mar 16\n#> 8 si 2.79 otra no Dom Feb 14\n#> 9 si 16.6 otra no Dom Abr 18\n#> 10 si 0.02 Chicago Independents si Dom Abr 15\n#> # ℹ 7,490 more rows" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-2", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-2", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(taxi_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#el-set-de-prueba", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#el-set-de-prueba", + "title": "2 - Tu presupuesto de datos", + "section": "El set de prueba ", + "text": "El set de prueba \n🙈\n\nEn el set de prueba, hay 2500 entradas, con 7 columnas" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-3", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-3", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWhat is in this?\n\ntaxi_folds <- vfold_cv(taxi_train)\ntaxi_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n\n\nTalk about a list column, storing non-atomic types in dataframe" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno-1", + "title": "2 - Tu presupuesto de datos", + "section": "Tu turno", + "text": "Tu turno\n\nSepara tus datos, el 20% tiene que ser para prueba\nTrata diferentes valores para set.seed() para ver como cambian los resultados\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-4", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-4", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(taxi_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [6400/1600]> Fold1\n#> 2 <split [6400/1600]> Fold2\n#> 3 <split [6400/1600]> Fold3\n#> 4 <split [6400/1600]> Fold4\n#> 5 <split [6400/1600]> Fold5" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-3", + "title": "2 - Tu presupuesto de datos", + "section": "Separando y “gastando” datos ", + "text": "Separando y “gastando” datos \n\nset.seed(123)\ntaxi_separar <- initial_split(taxi, prop = 0.8)\ntaxi_entrenar <- training(taxi_separar)\ntaxi_test <- testing(taxi_separar)\n\nnrow(taxi_entrenar)\n#> [1] 8000\nnrow(taxi_test)\n#> [1] 2000" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-5", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-5", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(taxi_train, strata = tip)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10\n\n\nStratification often helps, with very little downside" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-1", + "title": "2 - Tu presupuesto de datos", + "section": "", + "text": "flowchart TD\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000 \n vl[Validación]\n style vl fill:#E5E7FD,stroke:#666,color:#000 \n ad --> tr\n ad --> ts\n ad --> vl" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-6", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-6", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\ntaxi_folds <- vfold_cv(taxi_train, v = 10, strata = tip)\ntaxi_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10\n\n\nSet the seed when creating resamples" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-validación", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-validación", + "title": "2 - Tu presupuesto de datos", + "section": "Datos de validación", + "text": "Datos de validación\n\nset.seed(123)\ninitial_validation_split(taxi, prop = c(0.6, 0.2))\n#> <Training/Validation/Testing/Total>\n#> <6000/2000/2000/10000>" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", - "title": "4 - Evaluating models", - "section": "Fit our model to the resamples", - "text": "Fit our model to the resamples\n\ntaxi_res <- fit_resamples(taxi_wflow, taxi_folds)\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]>" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno-2", + "title": "2 - Tu presupuesto de datos", + "section": "Tu turno", + "text": "Tu turno\n\nExplora los datos en taxi_entrenar por tí mismo\n\n¿Cual es la distribución de los resultados de propina?\n¿Cual es la distribución de las variables numéricas, por ejemplo distancia?\n¿Como cambian los resultados de propina a través de las variables categóricas?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\ntaxi_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.915 10 0.00309 Preprocessor1_Model1\n#> 2 roc_auc binary 0.624 10 0.0105 Preprocessor1_Model1\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nWe can reliably measure performance using only the training data 🎉" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-2", + "title": "2 - Tu presupuesto de datos", + "section": "", + "text": "taxi_entrenar %>% \n ggplot(aes(x = propina)) +\n geom_bar()" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#comparing-metrics", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#comparing-metrics", - "title": "4 - Evaluating models", - "section": "Comparing metrics ", - "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\ntaxi_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.915 10\n#> 2 roc_auc 0.624 10\n\n\nThe ROC AUC previously was\n\n0.69 for the training set\n0.64 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-3", + "title": "2 - Tu presupuesto de datos", + "section": "", + "text": "taxi_entrenar %>% \n ggplot(aes(x = propina, fill = local)) +\n geom_bar() +\n scale_fill_viridis_d(end = .5)" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-1", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_taxi <- control_resamples(save_pred = TRUE)\ntaxi_res <- fit_resamples(taxi_wflow, taxi_folds, control = ctrl_taxi)\n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-4", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-4", + "title": "2 - Tu presupuesto de datos", + "section": "", + "text": "taxi_entrenar %>% \n ggplot(aes(x = hora, fill = propina)) +\n geom_bar()" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-2", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-2", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\ntaxi_preds <- collect_predictions(taxi_res)\ntaxi_preds\n#> # A tibble: 8,000 × 7\n#> id .pred_yes .pred_no .row .pred_class tip .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 Fold01 0.938 0.0615 14 yes yes Preprocessor1_Model1\n#> 2 Fold01 0.946 0.0544 19 yes yes Preprocessor1_Model1\n#> 3 Fold01 0.973 0.0269 33 yes yes Preprocessor1_Model1\n#> 4 Fold01 0.903 0.0971 43 yes yes Preprocessor1_Model1\n#> 5 Fold01 0.973 0.0269 74 yes yes Preprocessor1_Model1\n#> 6 Fold01 0.903 0.0971 103 yes yes Preprocessor1_Model1\n#> 7 Fold01 0.915 0.0851 104 yes no Preprocessor1_Model1\n#> 8 Fold01 0.903 0.0971 124 yes yes Preprocessor1_Model1\n#> 9 Fold01 0.667 0.333 126 yes yes Preprocessor1_Model1\n#> 10 Fold01 0.949 0.0510 128 yes yes Preprocessor1_Model1\n#> # ℹ 7,990 more rows" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-5", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-5", + "title": "2 - Tu presupuesto de datos", + "section": "", + "text": "taxi_entrenar %>% \n ggplot(aes(x = hora, fill = propina)) +\n geom_bar(position = \"fill\")" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-3", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-3", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\ntaxi_preds %>% \n group_by(id) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.905\n#> 2 Fold02 accuracy binary 0.925\n#> 3 Fold03 accuracy binary 0.926\n#> 4 Fold04 accuracy binary 0.915\n#> 5 Fold05 accuracy binary 0.902\n#> 6 Fold06 accuracy binary 0.912\n#> 7 Fold07 accuracy binary 0.906\n#> 8 Fold08 accuracy binary 0.91 \n#> 9 Fold09 accuracy binary 0.918\n#> 10 Fold10 accuracy binary 0.931\n#> # ℹ 20 more rows" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-6", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-6", + "title": "2 - Tu presupuesto de datos", + "section": "", + "text": "taxi_entrenar %>% \n ggplot(aes(x = distancia)) +\n geom_histogram(bins = 100) +\n facet_grid(vars(propina))" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#where-are-the-fitted-models", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#where-are-the-fitted-models", - "title": "4 - Evaluating models", - "section": "Where are the fitted models? ", - "text": "Where are the fitted models? \n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-7", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-7", + "title": "2 - Tu presupuesto de datos", + "section": "", + "text": "Estratificar la muestra separará los datos dentro los valores de respuesta\n\nBased on our EDA, we know that the source data contains fewer \"no\" tip values than \"yes\". We want to make sure we allot equal proportions of those responses so that both the training and testing data have enough of each to give accurate estimates." }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#bootstrapping", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#bootstrapping", - "title": "4 - Evaluating models", - "section": "Bootstrapping", - "text": "Bootstrapping" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#estratificar", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#estratificar", + "title": "2 - Tu presupuesto de datos", + "section": "Estratificar", + "text": "Estratificar\nUse strata = propina\n\nset.seed(123)\ntaxi_separar <- initial_split(taxi, prop = 0.8, strata = propina)\ntaxi_separar\n#> <Training/Testing/Total>\n#> <8000/2000/10000>" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#bootstrapping-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#bootstrapping-1", - "title": "4 - Evaluating models", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(taxi_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [8000/2902]> Bootstrap01\n#> 2 <split [8000/2916]> Bootstrap02\n#> 3 <split [8000/3004]> Bootstrap03\n#> 4 <split [8000/2979]> Bootstrap04\n#> 5 <split [8000/2961]> Bootstrap05\n#> 6 <split [8000/2962]> Bootstrap06\n#> 7 <split [8000/3026]> Bootstrap07\n#> 8 <split [8000/2926]> Bootstrap08\n#> 9 <split [8000/2972]> Bootstrap09\n#> 10 <split [8000/2972]> Bootstrap10\n#> # ℹ 15 more rows" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#estratificar-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#estratificar-1", + "title": "2 - Tu presupuesto de datos", + "section": "Estratificar", + "text": "Estratificar\nEstratificar la muestra usualmente ayuda, y con pocas consecuencias" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#expectativas-del-taller---donde-estamos", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#expectativas-del-taller---donde-estamos", + "title": "2 - Tu presupuesto de datos", + "section": "Expectativas del taller - Donde estamos", + "text": "Expectativas del taller - Donde estamos\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#fff,stroke:#eee,color:#ddd\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#fff,stroke:#eee,color:#ddd\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#fff,stroke:#eee,color:#ddd\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#fff,stroke:#eee,color:#ddd\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#fff,stroke:#eee,color:#ddd\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#fff,stroke:#eee,color:#ddd\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#fff,stroke:#eee,color:#ddd\n fm --> vm\n ts --> vm" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-3", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-3", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the functions)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" + "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", + "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", + "title": "Extras - workflowsets", + "section": "How can we compare multiple model workflows at once?", + "text": "How can we compare multiple model workflows at once?" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#monte-carlo-cross-validation", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#monte-carlo-cross-validation", - "title": "4 - Evaluating models", - "section": "Monte Carlo Cross-Validation ", - "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(taxi_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6000/2000]> Resample01\n#> 2 <split [6000/2000]> Resample02\n#> 3 <split [6000/2000]> Resample03\n#> 4 <split [6000/2000]> Resample04\n#> 5 <split [6000/2000]> Resample05\n#> 6 <split [6000/2000]> Resample06\n#> 7 <split [6000/2000]> Resample07\n#> 8 <split [6000/2000]> Resample08\n#> 9 <split [6000/2000]> Resample09\n#> 10 <split [6000/2000]> Resample10" + "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set", + "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec))\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#validation-set", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#validation-set", - "title": "4 - Evaluating models", - "section": "Validation set ", - "text": "Validation set \n\nset.seed(853)\ntaxi_val_split <- initial_validation_split(taxi, strata = tip)\nvalidation_set(taxi_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6000/2000]> validation\n\n\nA validation set is just another type of resample" + "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", + "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = taxi_folds)\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#random-forest-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#random-forest-1", - "title": "4 - Evaluating models", - "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this in Advanced tidymodels!), as long as there are enough trees" + "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", + "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = taxi_folds) %>%\n rank_results()\n#> # A tibble: 4 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_decision… Prepro… accura… 0.915 0.00309 10 formula deci… 1\n#> 2 formula_decision… Prepro… roc_auc 0.624 0.0105 10 formula deci… 1\n#> 3 formula_rand_for… Prepro… accura… 0.924 0.00326 10 formula rand… 2\n#> 4 formula_rand_for… Prepro… roc_auc 0.615 0.0151 10 formula rand… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#your-turn", + "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#your-turn", + "title": "Extras - workflowsets", + "section": "Your turn", + "text": "Your turn\n\nWhen do you think a workflow set would be useful?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model-1", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_wflow <- workflow(tip ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#looking-at-the-predictors", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#looking-at-the-predictors", + "title": "Extras - Recipes", + "section": "Looking at the predictors", + "text": "Looking at the predictors\n\ntaxi_train\n#> # A tibble: 8,000 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 17.2 Chicago Independents no Thu Feb 16\n#> 2 yes 0.88 City Service yes Thu Mar 8\n#> 3 yes 18.1 other no Mon Feb 18\n#> 4 yes 12.2 Chicago Independents no Sun Mar 21\n#> 5 yes 0.94 Sun Taxi yes Sat Apr 23\n#> 6 yes 17.5 Flash Cab no Fri Mar 12\n#> 7 yes 17.7 other no Sun Jan 6\n#> 8 yes 1.85 Taxicab Insurance Agency Llc no Fri Apr 12\n#> 9 yes 0.53 Sun Taxi no Tue Mar 18\n#> 10 yes 6.65 Taxicab Insurance Agency Llc no Sun Apr 11\n#> # ℹ 7,990 more rows" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-4", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-4", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#working-with-other-models", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#working-with-other-models", + "title": "Extras - Recipes", + "section": "Working with other models", + "text": "Working with other models\nSome models can’t handle non-numeric data\n\nLinear Regression\nK Nearest Neighbors\n\n\n\nSome models struggle if numeric predictors aren’t scaled\n\nK Nearest Neighbors\nAnything using gradient descent" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-4", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-4", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nctrl_taxi <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, taxi_folds, control = ctrl_taxi)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.923 10 0.00317 Preprocessor1_Model1\n#> 2 roc_auc binary 0.616 10 0.0147 Preprocessor1_Model1" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#types-of-needed-preprocessing", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#types-of-needed-preprocessing", + "title": "Extras - Recipes", + "section": "Types of needed preprocessing", + "text": "Types of needed preprocessing\n\nDo qualitative predictors require a numeric encoding?\nShould columns with a single unique value be removed?\nDoes the model struggle with missing data?\nDoes the model struggle with correlated predictors?\nShould predictors be centered and scaled?\nIs it helpful to transform predictors to be more symmetric?\n\n\nhttps://www.tmwr.org/pre-proc-table.html" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update-1", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing", + "title": "Extras - Recipes", + "section": "Two types of preprocessing", + "text": "Two types of preprocessing" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-final-fit", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-final-fit", - "title": "4 - Evaluating models", - "section": "The final fit ", - "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# taxi_split has train + test info\nfinal_fit <- last_fit(rf_wflow, taxi_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [8000/2000]> train/test split <tibble> <tibble> <tibble> <workflow>" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing-1", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing-1", + "title": "Extras - Recipes", + "section": "Two types of preprocessing", + "text": "Two types of preprocessing" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.914 Preprocessor1_Model1\n#> 2 roc_auc binary 0.638 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#general-definitions", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#general-definitions", + "title": "Extras - Recipes", + "section": "General definitions", + "text": "General definitions\n\nData preprocessing are the steps that you take to make your model successful.\nFeature engineering are what you do to the original predictors to make the model do the least work to perform great." }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-1", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-1", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 2,000 × 7\n#> id .pred_yes .pred_no .row .pred_class tip .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 train/test split 0.957 0.0426 4 yes yes Preprocessor1_Mo…\n#> 2 train/test split 0.938 0.0621 10 yes yes Preprocessor1_Mo…\n#> 3 train/test split 0.958 0.0416 19 yes yes Preprocessor1_Mo…\n#> 4 train/test split 0.894 0.106 23 yes yes Preprocessor1_Mo…\n#> 5 train/test split 0.943 0.0573 28 yes yes Preprocessor1_Mo…\n#> 6 train/test split 0.979 0.0213 34 yes yes Preprocessor1_Mo…\n#> 7 train/test split 0.954 0.0463 35 yes yes Preprocessor1_Mo…\n#> 8 train/test split 0.928 0.0722 38 yes yes Preprocessor1_Mo…\n#> 9 train/test split 0.985 0.0147 40 yes yes Preprocessor1_Mo…\n#> 10 train/test split 0.948 0.0523 42 yes no Preprocessor1_Mo…\n#> # ℹ 1,990 more rows" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#working-with-dates", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#working-with-dates", + "title": "Extras - Recipes", + "section": "Working with dates", + "text": "Working with dates\nDatetime variables are automatically converted to an integer if given as a raw predictor. To avoid this, it can be re-encoded as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nLeap year\nIndicators for holidays" + }, + { + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn", + "title": "Extras - Recipes", + "section": "Your turn", + "text": "Your turn\n\n\nWhat other transformations could we do with the raw time variable?\nRemember that the transformations are tied to the specific modeling problem.\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-2", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-2", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 8000 \n#> Number of independent variables: 6 \n#> Mtry: 2 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.07069778\n\n\nUse this for prediction on new data, like for deploying" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-transformations", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-transformations", + "title": "Extras - Recipes", + "section": "Two types of transformations", + "text": "Two types of transformations\n\n\n\nStatic\n\nSquare root, log, inverse\nDummies for known levels\nDate time extractions\n\n\nTrained\n\nCentering & scaling\nImputation\nPCA\nAnything for unknown factor levels\n\n\n\nTrained methods need to calculate sufficient information to be applied again." }, { - "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game", - "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game", - "title": "4 - Evaluating models", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#the-recipes-package", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#the-recipes-package", + "title": "Extras - Recipes", + "section": "The recipes package", + "text": "The recipes package\n\n\nModular + extensible\nWorks well with pipes ,|> and %>%\nDeferred evaluation\nIsolates test data from training data\nCan do things formulas can’t" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips", - "title": "2 - Your data budget", - "section": "Data on Chicago taxi trips", - "text": "Data on Chicago taxi trips\n\n\n\nThe city of Chicago releases anonymized trip-level data on taxi trips in the city.\nWe pulled a sample of 10,000 rides occurring in early 2022.\nType ?taxi to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/8322/taxi" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips-1", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips-1", - "title": "2 - Your data budget", - "section": "Data on Chicago taxi trips", - "text": "Data on Chicago taxi trips\n\n\n\nN = 10,000\nA nominal outcome, tip, with levels \"yes\" and \"no\"\nSeveral nominal variables like pickup & dropoff location, taxi ID, and payment type.\nSeveral numeric variables like trip length and fare subtotals.\n\n\n\n\n\nCredit: https://unsplash.com/photos/7_r85l4eht8\n\n\n“Fare subtotals” refers to the fare itself, tax, tolls, tip amount.\nActual variables in our data:\ntip: Whether the rider left a tip. A factor with levels “yes” and “no”.\ndistance: The trip distance, in odometer miles.\ncompany: The taxi company, as a factor. Companies that occurred few times were binned as “other”.\nlocal: Whether the trip started in the same community area as it began. See the source data for community area values.\ndow: The day of the week in which the trip began, as a factor.\nmonth: The month in which the trip began, as a factor.\nhour: The hour of the day in which the trip began, as a numeric." + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-1", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-1", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())\n\n\nStart by calling recipe() to denote the data source and variables used." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#checklist-for-predictors", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#checklist-for-predictors", - "title": "2 - Your data budget", - "section": "Checklist for predictors", - "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-2", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-2", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())\n\n\nSpecify what actions to take by adding step_*()s." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips-2", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips-2", - "title": "2 - Your data budget", - "section": "Data on Chicago taxi trips", - "text": "Data on Chicago taxi trips\n\nlibrary(tidymodels)\n\ntaxi\n#> # A tibble: 10,000 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 17.2 Chicago Independents no Thu Feb 16\n#> 2 yes 0.88 City Service yes Thu Mar 8\n#> 3 yes 18.1 other no Mon Feb 18\n#> 4 yes 20.7 Chicago Independents no Mon Apr 8\n#> 5 yes 12.2 Chicago Independents no Sun Mar 21\n#> 6 yes 0.94 Sun Taxi yes Sat Apr 23\n#> 7 yes 17.5 Flash Cab no Fri Mar 12\n#> 8 yes 17.7 other no Sun Jan 6\n#> 9 yes 1.85 Taxicab Insurance Agency Llc no Fri Apr 12\n#> 10 yes 1.47 City Service no Tue Mar 14\n#> # ℹ 9,990 more rows" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-3", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-3", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%   step_normalize(all_numeric_predictors())\n\n\nUse {tidyselect} and recipes-specific selectors to denote affected variables." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe", + "title": "Extras - Recipes", + "section": "Using a recipe", + "text": "Using a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%   step_normalize(all_numeric_predictors())\n\n\nSave the recipe we like so that we can use it in various places, e.g., with different models." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-1", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-1", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe-with-workflows", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe-with-workflows", + "title": "Extras - Recipes", + "section": "Using a recipe with workflows", + "text": "Using a recipe with workflows\nRecipes are typically combined with a model in a workflow() object:\n\n\ntaxi_wflow <- workflow() %>%\n  add_recipe(taxi_rec) %>%\n  add_model(linear_reg())" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-2", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-2", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#recipes-are-estimated", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#recipes-are-estimated", + "title": "Extras - Recipes", + "section": "Recipes are estimated", + "text": "Recipes are estimated\nEvery preprocessing step in a recipe that involved calculations uses the training set. For example:\n\nLevels of a factor\nDetermination of zero-variance\nNormalization\nFeature extraction\n\nOnce a recipe is added to a workflow, this occurs when fit() is called." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#debugging-a-recipe", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#debugging-a-recipe", + "title": "Extras - Recipes", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. taxi_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4.\n\n\n\n\nAnother function, bake(), is analogous to predict(), and gives you the processed data back." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-initial-split", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-initial-split", - "title": "2 - Your data budget", - "section": "The initial split ", - "text": "The initial split \n\nset.seed(123)\ntaxi_split <- initial_split(taxi)\ntaxi_split\n#> <Training/Testing/Total>\n#> <7500/2500/10000>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn-1", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn-1", + "title": "Extras - Recipes", + "section": "Your turn", + "text": "Your turn\n\n\nTake the recipe and prep() then bake() it to see what the resulting data set looks like.\nTry removing steps to see how the result changes.\n\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#what-is-set.seed", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#what-is-set.seed", - "title": "2 - Your data budget", - "section": "What is set.seed()?", - "text": "What is set.seed()?\nTo create that split of the data, R generates “pseudo-random” numbers: while they are made to behave like random numbers, their generation is deterministic give a “seed”.\nThis allows us to reproduce results by setting that seed.\nWhich seed you pick doesn’t matter, as long as you don’t try a bunch of seeds and pick the one that gives you the best performance." + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#printing-a-recipe", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#printing-a-recipe", + "title": "Extras - Recipes", + "section": "Printing a recipe", + "text": "Printing a recipe\n\ntaxi_rec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 6\n#> \n#> ── Operations\n#> • Unknown factor level assignment for: all_nominal_predictors()\n#> • Dummy variables from: all_nominal_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Log transformation on: distance\n#> • Centering and scaling for: all_numeric_predictors()" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#accessing-the-data", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#accessing-the-data", - "title": "2 - Your data budget", - "section": "Accessing the data ", - "text": "Accessing the data \n\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#prepping-a-recipe", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#prepping-a-recipe", + "title": "Extras - Recipes", + "section": "Prepping a recipe", + "text": "Prepping a recipe\n\nprep(taxi_rec)\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 6\n#> \n#> ── Training information\n#> Training data contained 8000 data points and no incomplete rows.\n#> \n#> ── Operations\n#> • Unknown factor level assignment for: company and local, ... |\n#> Trained\n#> • Dummy variables from: company, local, dow, month | Trained\n#> • Zero variance filter removed: company_unknown, ... | Trained\n#> • Log transformation on: distance | Trained\n#> • Centering and scaling for: distance and hour, ... | Trained" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-training-set", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-training-set", - "title": "2 - Your data budget", - "section": "The training set", - "text": "The training set\n\ntaxi_train\n#> # A tibble: 7,500 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 0.7 Taxi Affiliation Services yes Tue Mar 18\n#> 2 yes 0.99 Sun Taxi yes Tue Jan 8\n#> 3 yes 1.78 other no Sat Mar 22\n#> 4 yes 0 Taxi Affiliation Services yes Wed Apr 15\n#> 5 yes 0 Taxi Affiliation Services no Sun Jan 21\n#> 6 yes 2.3 other no Sat Apr 21\n#> 7 yes 6.35 Sun Taxi no Wed Mar 16\n#> 8 yes 2.79 other no Sun Feb 14\n#> 9 yes 16.6 other no Sun Apr 18\n#> 10 yes 0.02 Chicago Independents yes Sun Apr 15\n#> # ℹ 7,490 more rows" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#baking-a-recipe", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#baking-a-recipe", + "title": "Extras - Recipes", + "section": "Baking a recipe", + "text": "Baking a recipe\n\nprep(taxi_rec) %>%\n bake(new_data = taxi_train)\n#> # A tibble: 8,000 × 19\n#> distance hour tip company_City.Service company_Flash.Cab company_Sun.Taxi company_Taxi.Affiliatio…¹ company_Taxicab.Insu…² company_other local_no dow_Mon dow_Tue dow_Wed dow_Thu dow_Fri dow_Sat\n#> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 1.38 0.418 yes -0.366 -0.333 -0.403 -0.450 -0.379 -0.609 0.484 -0.396 -0.441 -0.461 2.01 -0.428 -0.317\n#> 2 -0.729 -1.42 yes 2.73 -0.333 -0.403 -0.450 -0.379 -0.609 -2.07 -0.396 -0.441 -0.461 2.01 -0.428 -0.317\n#> 3 1.42 0.877 yes -0.366 -0.333 -0.403 -0.450 -0.379 1.64 0.484 2.53 -0.441 -0.461 -0.497 -0.428 -0.317\n#> 4 1.11 1.57 yes -0.366 -0.333 -0.403 -0.450 -0.379 -0.609 0.484 -0.396 -0.441 -0.461 -0.497 -0.428 -0.317\n#> 5 -0.694 2.03 yes -0.366 -0.333 2.48 -0.450 -0.379 -0.609 -2.07 -0.396 -0.441 -0.461 -0.497 -0.428 3.15 \n#> 6 1.39 -0.502 yes -0.366 3.01 -0.403 -0.450 -0.379 -0.609 0.484 -0.396 -0.441 -0.461 -0.497 2.34 -0.317\n#> 7 1.40 -1.88 yes -0.366 -0.333 -0.403 -0.450 -0.379 1.64 0.484 -0.396 -0.441 -0.461 -0.497 -0.428 -0.317\n#> 8 -0.289 -0.502 yes -0.366 -0.333 -0.403 -0.450 2.64 -0.609 0.484 -0.396 -0.441 -0.461 -0.497 2.34 -0.317\n#> 9 -0.971 0.877 yes -0.366 -0.333 2.48 -0.450 -0.379 -0.609 0.484 -0.396 2.27 -0.461 -0.497 -0.428 -0.317\n#> 10 0.631 -0.732 yes -0.366 -0.333 -0.403 -0.450 2.64 -0.609 0.484 -0.396 -0.441 -0.461 -0.497 -0.428 -0.317\n#> # ℹ 7,990 more rows\n#> # ℹ abbreviated names: ¹​company_Taxi.Affiliation.Services, ²​company_Taxicab.Insurance.Agency.Llc\n#> # ℹ 3 more variables: month_Feb <dbl>, month_Mar <dbl>, month_Apr <dbl>" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-test-set", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-test-set", - "title": "2 - Your data budget", - "section": "The test set ", - "text": "The test set \n🙈\n\nThere are 2500 rows and 7 columns in the test set." + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\nOnce a recipe as been estimated, there are various bits of information saved in it.\n\nThe tidy() function can be used to get specific results from the recipe." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn-1", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn-1", - "title": "2 - Your data budget", + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn-2", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn-2", + "title": "Extras - Recipes", "section": "Your turn", - "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n−+\n05:00" + "text": "Your turn\n\nTake a prepped recipe and use the tidy() function on it.\nUse the number argument to inspect different steps.\n\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-3", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-3", - "title": "2 - Your data budget", - "section": "Data splitting and spending ", - "text": "Data splitting and spending \n\nset.seed(123)\ntaxi_split <- initial_split(taxi, prop = 0.8)\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)\n\nnrow(taxi_train)\n#> [1] 8000\nnrow(taxi_test)\n#> [1] 2000" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe-1", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe-1", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\n\nprep(taxi_rec) %>%\n tidy()\n#> # A tibble: 5 × 6\n#> number operation type trained skip id \n#> <int> <chr> <chr> <lgl> <lgl> <chr> \n#> 1 1 step unknown TRUE FALSE unknown_NTmu5 \n#> 2 2 step dummy TRUE FALSE dummy_cT3Uy \n#> 3 3 step zv TRUE FALSE zv_z22dk \n#> 4 4 step log TRUE FALSE log_QQ1iw \n#> 5 5 step normalize TRUE FALSE normalize_3cTJb" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#validation-set", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#validation-set", - "title": "2 - Your data budget", - "section": "Validation set", - "text": "Validation set\n\nset.seed(123)\ninitial_validation_split(taxi, prop = c(0.6, 0.2))\n#> <Training/Validation/Testing/Total>\n#> <6000/2000/2000/10000>" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe-2", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe-2", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\n\nprep(taxi_rec) %>%\n tidy(number = 2)\n#> # A tibble: 20 × 3\n#> terms columns id \n#> <chr> <chr> <chr> \n#> 1 company City Service dummy_cT3Uy\n#> 2 company Flash Cab dummy_cT3Uy\n#> 3 company Sun Taxi dummy_cT3Uy\n#> 4 company Taxi Affiliation Services dummy_cT3Uy\n#> 5 company Taxicab Insurance Agency Llc dummy_cT3Uy\n#> 6 company other dummy_cT3Uy\n#> 7 company unknown dummy_cT3Uy\n#> 8 local no dummy_cT3Uy\n#> 9 local unknown dummy_cT3Uy\n#> 10 dow Mon dummy_cT3Uy\n#> 11 dow Tue dummy_cT3Uy\n#> 12 dow Wed dummy_cT3Uy\n#> 13 dow Thu dummy_cT3Uy\n#> 14 dow Fri dummy_cT3Uy\n#> 15 dow Sat dummy_cT3Uy\n#> 16 dow unknown dummy_cT3Uy\n#> 17 month Feb dummy_cT3Uy\n#> 18 month Mar dummy_cT3Uy\n#> 19 month Apr dummy_cT3Uy\n#> 20 month unknown dummy_cT3Uy" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn-2", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn-2", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nExplore the taxi_train data on your own!\n\nWhat’s the distribution of the outcome, tip?\nWhat’s the distribution of numeric variables like distance?\nHow does tip differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe-in-tidymodels", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe-in-tidymodels", + "title": "Extras - Recipes", + "section": "Using a recipe in tidymodels", + "text": "Using a recipe in tidymodels\nThe recommended way to use a recipe in tidymodels is to use it as part of a workflow().\n\ntaxi_wflow <- workflow() %>% \n add_recipe(taxi_rec) %>% \n add_model(linear_reg())\n\nWhen used in this way, you don’t need to worry about prep() and bake() as it is handled for you." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-2", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-2", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n ggplot(aes(x = tip)) +\n geom_bar()" + "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#more-information", + "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#more-information", + "title": "Extras - Recipes", + "section": "More information", + "text": "More information\n\nhttps://recipes.tidymodels.org/\nhttps://www.tmwr.org/recipes.html" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-3", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-3", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n ggplot(aes(x = tip, fill = local)) +\n geom_bar() +\n scale_fill_viridis_d(end = .5)" + "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#tuning-parameters", + "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#tuning-parameters", + "title": "5 - Tuning models", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-4", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-4", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n ggplot(aes(x = hour, fill = tip)) +\n geom_bar()" + "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters", + "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters", + "title": "5 - Tuning models", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-5", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-5", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n ggplot(aes(x = hour, fill = tip)) +\n geom_bar(position = \"fill\")" + "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters-1", + "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters-1", + "title": "5 - Tuning models", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-6", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-6", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n ggplot(aes(x = distance)) +\n geom_histogram(bins = 100) +\n facet_grid(vars(tip))" + "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#specifying-tuning-parameters", + "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#specifying-tuning-parameters", + "title": "5 - Tuning models", + "section": "Specifying tuning parameters", + "text": "Specifying tuning parameters\nLet’s take our previous random forest workflow and tag for tuning the minimum number of data points in each node:\n\nrf_spec <- rand_forest(min_n = tune()) %>% \n set_mode(\"classification\")\n\nrf_wflow <- workflow(tip ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-7", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-7", - "title": "2 - Your data budget", - "section": "", - "text": "Stratified sampling would split within response values\n\nBased on our EDA, we know that the source data contains fewer \"no\" tip values than \"yes\". We want to make sure we allot equal proportions of those responses so that both the training and testing data have enough of each to give accurate estimates." + "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#try-out-multiple-values", + "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#try-out-multiple-values", + "title": "5 - Tuning models", + "section": "Try out multiple values", + "text": "Try out multiple values\ntune_grid() works similar to fit_resamples() but covers multiple parameter values:\n\nset.seed(22)\nrf_res <- tune_grid(\n rf_wflow,\n taxi_folds,\n grid = 5\n)" + }, + { + "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#compare-results", + "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#compare-results", + "title": "5 - Tuning models", + "section": "Compare results", + "text": "Compare results\nInspecting results and selecting the best-performing hyperparameter(s):\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 33 roc_auc binary 0.623 10 0.0149 Preprocessor1_Model1\n#> 2 31 roc_auc binary 0.622 10 0.0154 Preprocessor1_Model3\n#> 3 21 roc_auc binary 0.620 10 0.0149 Preprocessor1_Model4\n#> 4 13 roc_auc binary 0.617 10 0.0137 Preprocessor1_Model5\n#> 5 6 roc_auc binary 0.611 10 0.0156 Preprocessor1_Model2\n\nbest_parameter <- select_best(rf_res)\nbest_parameter\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 33 Preprocessor1_Model1\n\ncollect_metrics() and autoplot() are also available." }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#stratification", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#stratification", - "title": "2 - Your data budget", - "section": "Stratification", - "text": "Stratification\nUse strata = tip\n\nset.seed(123)\ntaxi_split <- initial_split(taxi, prop = 0.8, strata = tip)\ntaxi_split\n#> <Training/Testing/Total>\n#> <8000/2000/10000>" + "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#the-final-fit", + "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#the-final-fit", + "title": "5 - Tuning models", + "section": "The final fit", + "text": "The final fit\n\nrf_wflow <- finalize_workflow(rf_wflow, best_parameter)\n\nfinal_fit <- last_fit(rf_wflow, taxi_split) \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.913 Preprocessor1_Model1\n#> 2 roc_auc binary 0.648 Preprocessor1_Model1" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#stratification-1", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#stratification-1", - "title": "2 - Your data budget", - "section": "Stratification", - "text": "Stratification\nStratification often helps, with very little downside" + "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#your-turn", + "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#your-turn", + "title": "5 - Tuning models", + "section": "Your turn", + "text": "Your turn\n\nModify your model workflow to tune one or more parameters.\nUse grid search to find the best parameter(s).\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-whole-game---status-update", - "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-whole-game---status-update", - "title": "2 - Your data budget", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglm for generalized linear model (e.g. logistic regression)\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets" }, { - "objectID": "archive/2023-09-posit-conf/index.html", - "href": "archive/2023-09-posit-conf/index.html", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels offered at posit::conf(2023). The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to tidymodels and Advanced tidymodels courses.\nIntroduction to tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and basic pre-processing with recipes. Time permitting, you’ll be introduced to model optimization using the tune package. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nAdvanced tidymodels will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/index.html#welcome", - "href": "archive/2023-09-posit-conf/index.html#welcome", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels offered at posit::conf(2023). The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to tidymodels and Advanced tidymodels courses.\nIntroduction to tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and basic pre-processing with recipes. Time permitting, you’ll be introduced to model optimization using the tune package. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nAdvanced tidymodels will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-1", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-1", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" }, { - "objectID": "archive/2023-09-posit-conf/index.html#is-this-workshop-for-me", - "href": "archive/2023-09-posit-conf/index.html#is-this-workshop-for-me", - "title": "Machine learning with tidymodels", - "section": "Is this workshop for me? ", - "text": "Is this workshop for me? \nDepending on your background, one of Introduction to tidymodels or Advanced tidymodels might serve you better than the other.\n\nIntroduction to tidymodels\nThis workshop is for you if you:\n\nare comfortable using tidyverse packages to read data into R, transform and reshape data, and make a variety of graphs, and\nhave had some exposure to basic statistical concepts such as linear models, residuals, etc.\n\nIntermediate or expert familiarity with modeling or machine learning is not required. Interested students who have intermediate or expert familiarity with modeling or machine learning may be interested in the Advanced tidymodels workshop.\n\n\nAdvanced tidymodels\nThis workshop is for you if you:\n\nhave the prerequisite skills listed for the Introduction to tidymodels workshops,\nhave used tidymodels packages like recipes, rsample, and parsnip, and\nhave some experience with evaluating statistical models using resampling techniques like v-fold cross-validation or the bootstrap.\n\nParticipants who are new to tidymodels or machine learning will benefit from taking the Introduction to tidymodels workshop before joining this one. Participants who have completed the “Introduction to tidymodels” workshop will be well-prepared for this course." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-2", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-2", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/index.html#preparation", - "href": "archive/2023-09-posit-conf/index.html#preparation", - "title": "Machine learning with tidymodels", - "section": "Preparation", - "text": "Preparation\nThe process to set up your computer for either workshop will look the same. Please join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2022.02), available at https://posit.co/download/rstudio-desktop/\nThe following R packages, which you can install from the R console:\n\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\",\n \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\",\n \"splines2\", \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \n \"vetiver\", \"remotes\")\n\ninstall.packages(pkgs)\n\nIf you’re a Windows user and encounter an error message during installation noting a missing Rtools installation, install Rtools using the installer linked here." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-3", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-3", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" }, { - "objectID": "archive/2023-09-posit-conf/index.html#slides", - "href": "archive/2023-09-posit-conf/index.html#slides", - "title": "Machine learning with tidymodels", - "section": "Slides", - "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. There are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to tidymodels\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n05: Tuning models\n06: Wrapping up\n\n\n\nAdvanced tidymodels\n\n01: Introduction\n02: Feature engineering using recipes\n03: Tuning hyperparameters (grid search)\n04: Grid search via racing\n05: Iterative search\n06: Wrapping up\n\n\n\nExtra content (time permitting)\n\nIntro: Time-based splitting\nIntro: Using workflowsets\nIntro: Using recipes\nAdvanced: Transit case study (includes stacking)\nAdvanced: Effect encoding\nAdvanced: Model deployment\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-4", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-4", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" }, { - "objectID": "archive/2023-09-posit-conf/index.html#code", - "href": "archive/2023-09-posit-conf/index.html#code", - "title": "Machine learning with tidymodels", - "section": "Code", - "text": "Code\nQuarto files for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-5", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-5", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/index.html#past-workshops", - "href": "archive/2023-09-posit-conf/index.html#past-workshops", - "title": "Machine learning with tidymodels", - "section": "Past workshops", - "text": "Past workshops\n\nJuly 2022 at rstudio::conf()\nAugust 2022 in Reykjavik\nJuly 2023 at the New York R Conference" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-6", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-6", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" }, { - "objectID": "archive/2023-09-posit-conf/index.html#acknowledgments", - "href": "archive/2023-09-posit-conf/index.html#acknowledgments", - "title": "Machine learning with tidymodels", - "section": "Acknowledgments", - "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-7", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-7", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2023-09-posit-conf/index.html#reuse-and-licensing", - "href": "archive/2023-09-posit-conf/index.html#reuse-and-licensing", - "title": "Machine learning with tidymodels", - "section": "Reuse and licensing", - "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-8", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-8", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#chicago-l-train-data", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#chicago-l-train-data", - "title": "Case Study on Transportation", - "section": "Chicago L-Train data", - "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-1", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-1", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a logistic regression model.\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/\n\n\nExtension/Challenge: Edit this code to use a different model. For example, try using a conditional inference tree as implemented in the partykit package by changing the engine - or try an entirely different model type!\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#predictors", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#predictors", - "title": "Case Study on Transportation", - "section": "Predictors", - "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#models-well-be-using-today", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#models-well-be-using-today", + "title": "3 - What makes a model?", + "section": "Models we’ll be using today", + "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#l-train-locations", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#l-train-locations", - "title": "Case Study on Transportation", - "section": "L Train Locations", - "text": "L Train Locations" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#your-turn-explore-the-data", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#your-turn-explore-the-data", - "title": "Case Study on Transportation", - "section": "Your turn: Explore the Data", - "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n−+\n05:00" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-1", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-1", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#splitting-with-chicago-data", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#splitting-with-chicago-data", - "title": "Case Study on Transportation", - "section": "Splitting with Chicago data ", - "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-2", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-2", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nFind a sigmoid line that separates the two classes" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#time-series-resampling", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#time-series-resampling", - "title": "Case Study on Transportation", - "section": "Time series resampling", - "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#rolling-forecast-origin-resampling", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#rolling-forecast-origin-resampling", - "title": "Case Study on Transportation", - "section": "Rolling forecast origin resampling", - "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees-1", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees-1", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees-2", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees-2", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-1", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-1", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "title": "3 - What makes a model?", + "section": "All models are wrong, but some are useful!", + "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-2", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-2", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "title": "3 - What makes a model?", + "section": "Workflows bind preprocessors and models", + "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-3", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-3", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#what-is-wrong-with-this", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#what-is-wrong-with-this", + "title": "3 - What makes a model?", + "section": "What is wrong with this?", + "text": "What is wrong with this?" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-4", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-4", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#why-a-workflow", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#why-a-workflow", + "title": "3 - What makes a model?", + "section": "Why a workflow()? ", + "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering in Advanced tidymodels!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#our-resampling-object", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#our-resampling-object", - "title": "Case Study on Transportation", - "section": "Our resampling object ", - "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-1", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-1", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\ntree_spec %>% \n fit(tip ~ ., data = taxi_train) \n#> parsnip model object\n#> \n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 yes (0.92300000 0.07700000) \n#> 2) distance>=14.12 2041 68 yes (0.96668300 0.03331700) *\n#> 3) distance< 14.12 5959 548 yes (0.90803826 0.09196174) \n#> 6) distance< 5.275 5419 450 yes (0.91695885 0.08304115) *\n#> 7) distance>=5.275 540 98 yes (0.81851852 0.18148148) \n#> 14) company=Chicago Independents,City Service,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc,other 478 68 yes (0.85774059 0.14225941) *\n#> 15) company=Flash Cab 62 30 yes (0.51612903 0.48387097) \n#> 30) dow=Thu 12 2 yes (0.83333333 0.16666667) *\n#> 31) dow=Sun,Mon,Tue,Wed,Fri,Sat 50 22 no (0.44000000 0.56000000) \n#> 62) distance>=11.77 14 4 yes (0.71428571 0.28571429) *\n#> 63) distance< 11.77 36 12 no (0.33333333 0.66666667) *" }, - { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#feature-engineering-with-recipes", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#feature-engineering-with-recipes", - "title": "Case Study on Transportation", - "section": "Feature engineering with recipes ", - "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" + { + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-2", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-2", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\nworkflow() %>%\n add_formula(tip ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = taxi_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 yes (0.92300000 0.07700000) \n#> 2) distance>=14.12 2041 68 yes (0.96668300 0.03331700) *\n#> 3) distance< 14.12 5959 548 yes (0.90803826 0.09196174) \n#> 6) distance< 5.275 5419 450 yes (0.91695885 0.08304115) *\n#> 7) distance>=5.275 540 98 yes (0.81851852 0.18148148) \n#> 14) company=Chicago Independents,City Service,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc,other 478 68 yes (0.85774059 0.14225941) *\n#> 15) company=Flash Cab 62 30 yes (0.51612903 0.48387097) \n#> 30) dow=Thu 12 2 yes (0.83333333 0.16666667) *\n#> 31) dow=Sun,Mon,Tue,Wed,Fri,Sat 50 22 no (0.44000000 0.56000000) \n#> 62) distance>=11.77 14 4 yes (0.71428571 0.28571429) *\n#> 63) distance< 11.77 36 12 no (0.33333333 0.66666667) *" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe", - "title": "Case Study on Transportation", - "section": "A recipe", - "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-3", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-3", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\nworkflow(tip ~ ., tree_spec) %>% \n fit(data = taxi_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 yes (0.92300000 0.07700000) \n#> 2) distance>=14.12 2041 68 yes (0.96668300 0.03331700) *\n#> 3) distance< 14.12 5959 548 yes (0.90803826 0.09196174) \n#> 6) distance< 5.275 5419 450 yes (0.91695885 0.08304115) *\n#> 7) distance>=5.275 540 98 yes (0.81851852 0.18148148) \n#> 14) company=Chicago Independents,City Service,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc,other 478 68 yes (0.85774059 0.14225941) *\n#> 15) company=Flash Cab 62 30 yes (0.51612903 0.48387097) \n#> 30) dow=Thu 12 2 yes (0.83333333 0.16666667) *\n#> 31) dow=Sun,Mon,Tue,Wed,Fri,Sat 50 22 no (0.44000000 0.56000000) \n#> 62) distance>=11.77 14 4 yes (0.71428571 0.28571429) *\n#> 63) distance< 11.77 36 12 no (0.33333333 0.66666667) *" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-2", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-2", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code to make a workflow with your own model of choice.\n\nExtension/Challenge: Other than formulas, what kinds of preprocessors are supported?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-1", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-1", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#predict-with-your-model", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#predict-with-your-model", + "title": "3 - What makes a model?", + "section": "Predict with your model ", + "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(tip ~ ., tree_spec) %>% \n fit(data = taxi_train)" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-2", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-2", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-3", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-3", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = taxi_test)\nWhat do you get?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---remove-constant-columns", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---remove-constant-columns", - "title": "Case Study on Transportation", - "section": "A recipe - remove constant columns ", - "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE) %>% \n step_zv(all_nominal_predictors())" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-4", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-4", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = taxi_test)\nWhat do you get?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---handle-correlations", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---handle-correlations", - "title": "Case Study on Transportation", - "section": "A recipe - handle correlations ", - "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#make-some-models", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#make-some-models", - "title": "Case Study on Transportation", - "section": "Make some models ", - "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) %>% set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) %>% \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-1", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-1", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\n⚠️ Never predict() with any extracted components!\n\nroundint = FALSE is only to quiet a warning" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#process-them-on-the-resamples", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#process-them-on-the-resamples", - "title": "Case Study on Transportation", - "section": "Process them on the resamples", - "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set %>% \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-2", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-2", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#how-do-the-results-look", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#how-do-the-results-look", - "title": "Case Study on Transportation", - "section": "How do the results look?", - "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 31 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.798 0.104 16 recipe cubis… 1\n#> 2 pca_cubist Preprocessor3_Model3 mae 0.978 0.110 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor4_Model2 mae 0.983 0.122 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model1 mae 0.991 0.127 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor3_Model2 mae 0.991 0.113 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor2_Model2 mae 1.02 0.118 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor1_Model3 mae 1.05 0.134 16 recipe cubis… 7\n#> 8 basic_cubist Preprocessor1_Model8 mae 1.07 0.115 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model7 mae 1.07 0.112 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model6 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 21 more rows" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-5", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-5", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\n\nExtract the model engine object from your fitted workflow and check it out.\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#plot-the-results", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#plot-the-results", - "title": "Case Study on Transportation", - "section": "Plot the results ", - "text": "Plot the results \n\nautoplot(chi_res)" + "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#the-whole-game---status-update", + "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#the-whole-game---status-update", + "title": "3 - What makes a model?", + "section": "The whole game - status update", + "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#pull-out-specific-results", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#pull-out-specific-results", - "title": "Case Study on Transportation", - "section": "Pull out specific results ", - "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n autoplot()" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#workshop-policies", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#workshop-policies", + "title": "1 - Introduction", + "section": "Workshop policies", + "text": "Workshop policies\n\nPlease do not photograph people wearing red lanyards\nThere are gender-neutral bathrooms located are among the Grand Suite Bathrooms\nThere are two meditation/prayer rooms: Grand Suite 2A and 2B\nA lactation room is located in Grand Suite 1\nThe meditation/prayer and lactation rooms are open\nSun - Tue 7:30am - 7:00pm, Wed 8:00am - 6:00pm" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#workshop-policies-1", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#workshop-policies-1", + "title": "1 - Introduction", + "section": "Workshop policies", + "text": "Workshop policies\n\nPlease review the code of conduct and COVID policies, which apply to all workshops: https://posit.co/code-of-conduct/.\nCoC site has info on how to report a problem (in person, email, phone)" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-1", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-1", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#who-are-you", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-2", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-2", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#who-are-tidymodels", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nSilvia Canelón is TAing today!\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-3", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-3", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#asking-for-help", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#asking-for-help", + "title": "1 - Introduction", + "section": "Asking for help", + "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-4", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-4", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#section-2", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#section-2", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-5", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-5", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#plan-for-this-workshop", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Plan for this workshop", + "text": "Plan for this workshop\n\nYour data budget\nWhat makes a model\nEvaluating models\nTuning models\n\n\nThis workshop will well-prepare folks going on to the Advanced tidymodels workshop, which will cover feature engineering and much more on hyperparameter tuning." }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#building-a-model-stack", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#building-a-model-stack", - "title": "Case Study on Transportation", - "section": "Building a model stack ", - "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#section-3", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#section-3", + "title": "1 - Introduction", + "section": "", + "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free):\nCheck the workshop channel on Discord for the link!" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#start-the-stack-and-add-members", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#start-the-stack-and-add-members", - "title": "Case Study on Transportation", - "section": "Start the stack and add members ", - "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() %>% \n add_candidates(chi_res)" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#estimate-weights-for-each-candidate", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#estimate-weights-for-each-candidate", - "title": "Case Study on Transportation", - "section": "Estimate weights for each candidate ", - "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack)\n\nchi_stack_res\n#> # A tibble: 5 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.343\n#> 2 pca_cubist_3_2 cubist_rules 0.236\n#> 3 basic_cubist_1_4 cubist_rules 0.189\n#> 4 pca_lm_4_1 linear_reg 0.163\n#> 5 pca_cubist_3_3 cubist_rules 0.109" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning-1", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning-1", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#how-did-it-do", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#how-did-it-do", - "title": "Case Study on Transportation", - "section": "How did it do? ", - "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning-2", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning-2", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#what-does-it-use", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#what-does-it-use", - "title": "Case Study on Transportation", - "section": "What does it use? ", - "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#your-turn", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#your-turn", + "title": "1 - Introduction", + "section": "Your turn", + "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#fit-the-required-candidate-models", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#fit-the-required-candidate-models", - "title": "Case Study on Transportation", - "section": "Fit the required candidate models", - "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-tidymodels", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-tidymodels", + "title": "1 - Introduction", + "section": "What is tidymodels? ", + "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.1.1 ──\n#> ✔ broom 1.0.5 ✔ rsample 1.2.0\n#> ✔ dials 1.2.0 ✔ tibble 3.2.1\n#> ✔ dplyr 1.1.3 ✔ tidyr 1.3.0\n#> ✔ infer 1.0.5 ✔ tune 1.1.2\n#> ✔ modeldata 1.2.0 ✔ workflows 1.1.3\n#> ✔ parsnip 1.1.1 ✔ workflowsets 1.0.1\n#> ✔ purrr 1.0.2 ✔ yardstick 1.2.0\n#> ✔ recipes 1.0.8\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Learn how to get started at https://www.tidymodels.org/start/" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#the-test-set-best-cubist-model", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#the-test-set-best-cubist-model", - "title": "Case Study on Transportation", - "section": "The test set: best Cubist model ", - "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n select_best()\n\ncubist_res <- \n chi_res %>% \n extract_workflow(\"pca_cubist\") %>% \n finalize_workflow(best_cubist) %>% \n last_fit(split = chi_split, metrics = metric_set(mae))" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\nRoadmap for today\nMinimal version of predictive modeling process\nFeature engineering and tuning as iterative extensions" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#the-test-set-stack-ensemble", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#the-test-set-stack-ensemble", - "title": "Case Study on Transportation", - "section": "The test set: stack ensemble", - "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) %>% \n bind_cols(chi_test)" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-1", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-1", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#compare-the-results", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#compare-the-results", - "title": "Case Study on Transportation", - "section": "Compare the results ", - "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.670 Preprocessor1_Model1\n\nstack_pred %>% mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.689" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-2", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-2", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." }, { - "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#plot-the-test-set", - "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#plot-the-test-set", - "title": "Case Study on Transportation", - "section": "Plot the test set ", - "text": "Plot the test set \n\n\nlibrary(probably)\ncubist_res %>% \n collect_predictions() %>% \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-3", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-3", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-09-posit-conf/annotations.html#section", - "href": "archive/2023-09-posit-conf/annotations.html#section", - "title": "Annotations", - "section": "👀", - "text": "👀\nThis page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-4", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-4", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-09-posit-conf/annotations.html#the-initial-split", - "href": "archive/2023-09-posit-conf/annotations.html#the-initial-split", - "title": "Annotations", - "section": "The initial split", - "text": "The initial split\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-5", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-5", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-09-posit-conf/annotations.html#what-is-wrong-with-this", - "href": "archive/2023-09-posit-conf/annotations.html#what-is-wrong-with-this", - "title": "Annotations", - "section": "What is wrong with this?", - "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-6", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-6", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-09-posit-conf/annotations.html#where-are-the-fitted-models", - "href": "archive/2023-09-posit-conf/annotations.html#where-are-the-fitted-models", - "title": "Annotations", - "section": "Where are the fitted models?", - "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-7", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-7", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-09-posit-conf/annotations.html#update-parameter-ranges", - "href": "archive/2023-09-posit-conf/annotations.html#update-parameter-ranges", - "title": "Annotations", - "section": "Update parameter ranges", - "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_spline_natural(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#lets-install-some-packages", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\",\n \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\",\n \"splines2\", \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \n \"vetiver\", \"remotes\")\n\ninstall.packages(pkgs)\n\n\n\n Or log in to Posit Cloud\nLink in our Discord channel!" }, { - "objectID": "archive/2023-09-posit-conf/annotations.html#early-stopping-for-boosted-trees", - "href": "archive/2023-09-posit-conf/annotations.html#early-stopping-for-boosted-trees", - "title": "Annotations", - "section": "Early stopping for boosted trees", - "text": "Early stopping for boosted trees\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the lightgbm package.\nbonsai has its own wrapper around (lightgbm::lgb.train()) called bonsai::train_lightgbm(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the lightgbm entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for lightgbm)." + "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#our-versions", + "href": "archive/2023-09-posit-conf/intro-01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nR version 4.2.2 (2022-10-31), Quarto (1.4.104)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.2.1\n\n\nbroom\n1.0.5\n\n\ndials\n1.2.0\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.3\n\n\nembed\n1.1.2\n\n\nfinetune\n1.1.0\n\n\nggplot2\n3.4.3\n\n\nlightgbm\n3.3.5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nlme4\n1.1-34\n\n\nmodeldata\n1.2.0\n\n\nparsnip\n1.1.1\n\n\nplumber\n1.2.1\n\n\nprobably\n1.0.2\n\n\npurrr\n1.0.2\n\n\nranger\n0.15.1\n\n\nrecipes\n1.0.8\n\n\nremotes\n2.4.2.1\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nrpart\n4.1.19\n\n\nrpart.plot\n3.1.1\n\n\nrsample\n1.2.0\n\n\nrules\n1.0.2\n\n\nscales\n1.2.1\n\n\nsplines2\n0.5.1\n\n\nstacks\n1.0.2\n\n\ntext2vec\n0.6.3\n\n\ntextrecipes\n1.0.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.1.1\n\n\ntidyr\n1.3.0\n\n\ntune\n1.1.2\n\n\nvetiver\n0.2.4\n\n\nworkflows\n1.1.3\n\n\nworkflowsets\n1.0.1\n\n\nyardstick\n1.2.0" }, { - "objectID": "archive/2023-09-posit-conf/annotations.html#per-agent-statistics", - "href": "archive/2023-09-posit-conf/annotations.html#per-agent-statistics", - "title": "Annotations", - "section": "Per-agent statistics", - "text": "Per-agent statistics\nThe effect encoding method essentially takes the effect of a variable, like agent, and makes a data column for that effect. In our example, affect of the agent on the ADR is quantified by a model and then added as a data column to be used in the model.\nSuppose agent Max has a single reservation in the data and it had an ADR of €200. If we used a naive estimate for Max’s effect, the model is being told that Max should always produce an effect of €200. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned agent Davis, who has taken 250 reservations with an average ADR of €100. Davis’s mean is more predictive because it is estimated with better data (i.e., more total reservations). Partial pooling leverages the entire data set and can borrow strength from all of the agents. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a agent’s data is of good quality, the partial pooling effect estimate is closer to the raw mean. Max’s data is not great and is “shrunk” towards the center of the overall average. Since there is so little known about Max’s reservation history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nSince this example has a numeric outcome, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" + "objectID": "archive/2023-09-posit-conf/extras-vetiver.html#deploying-a-model", + "href": "archive/2023-09-posit-conf/extras-vetiver.html#deploying-a-model", + "title": "Extras - Model deployment", + "section": "Deploying a model ", + "text": "Deploying a model \nWe have a decision tree, tree_fit, to model whether or not a taxi trip in Chicago included a tip or not.\nHow do we use our model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"taxi\")\nv\n#> \n#> ── taxi ─ <bundled_workflow> model for deployment \n#> A rpart classification modeling workflow using 6 features\n\nLearn more at https://vetiver.rstudio.com" }, { - "objectID": "archive/2023-09-posit-conf/annotations.html#agent-effects", - "href": "archive/2023-09-posit-conf/annotations.html#agent-effects", - "title": "Annotations", - "section": "Agent effects", - "text": "Agent effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage though." + "objectID": "archive/2023-09-posit-conf/extras-vetiver.html#deploy-your-model-1", + "href": "archive/2023-09-posit-conf/extras-vetiver.html#deploy-your-model-1", + "title": "Extras - Model deployment", + "section": "Deploy your model ", + "text": "Deploy your model \nHow do we use our model in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 4 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Users/simoncouch/Library/R/arm64/4.2/library/vetiver\n#> ├──/metadata (GET)\n#> ├──/ping (GET)\n#> ├──/predict (POST)\n#> └──/prototype (GET)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---setup", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---setup", - "title": "4 - Iterative Search", + "objectID": "archive/2023-09-posit-conf/extras-vetiver.html#your-turn", + "href": "archive/2023-09-posit-conf/extras-vetiver.html#your-turn", + "title": "Extras - Model deployment", + "section": "Your turn", + "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n−+\n05:00" + }, + { + "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#previously---setup", + "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#previously---setup", + "title": "Extras - Effect Encodings", "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---data-usage", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---data-usage", - "title": "4 - Iterative Search", + "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#previously---data-usage", + "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#previously---data-usage", + "title": "Extras - Effect Encodings", "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---boosting-model", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---boosting-model", - "title": "4 - Iterative Search", - "section": "Previously - Boosting Model", - "text": "Previously - Boosting Model\n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", + "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", + "title": "Extras - Effect Encodings", + "section": "What do we do with the agent and company data?", + "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#iterative-search", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#iterative-search", - "title": "4 - Iterative Search", - "section": "Iterative Search", - "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our boosted tree.\n\nWe could do something like:\nmae_pred <- lm(mae ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." + "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#per-agent-statistics", + "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#per-agent-statistics", + "title": "Extras - Effect Encodings", + "section": "Per-agent statistics", + "text": "Per-agent statistics" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#iterative-search-1", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#iterative-search-1", - "title": "4 - Iterative Search", - "section": "Iterative Search", - "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between MAE and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" + "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#what-is-an-effect-encoding", + "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#what-is-an-effect-encoding", + "title": "Extras - Effect Encodings", + "section": "What is an effect encoding?", + "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <fct> <int>\n#> 1 52.7 cynthia_worsley 1\n#> 2 51.8 carlos_bryant 2\n#> 3 53.8 lance_hitchcock 3\n#> 4 51.8 lance_hitchcock 4\n#> 5 46.8 cynthia_worsley 5\n#> 6 54.7 charles_najera 6\n#> 7 46.8 cynthia_worsley 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <dbl> <int>\n#> 1 52.7 88.5 1\n#> 2 51.8 89.5 2\n#> 3 53.8 79.8 3\n#> 4 51.8 79.8 4\n#> 5 46.8 88.5 5\n#> 6 54.7 109. 6\n#> 7 46.8 88.5 7\n\n\nThe agent column is replaced with an estimate of the ADR." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#a-large-grid", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#a-large-grid", - "title": "4 - Iterative Search", - "section": "A Large Grid", - "text": "A Large Grid" + "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#per-agent-statistics-again", + "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#per-agent-statistics-again", + "title": "Extras - Effect Encodings", + "section": "Per-agent statistics again", + "text": "Per-agent statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these means use partial pooling.\nPooling borrows strength across agents and shrinks extreme values towards the mean for agents with very few transations\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for agents with fewer reservations by shrinking the estimate to the overall ADR mean" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#a-three-point-grid", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#a-three-point-grid", - "title": "4 - Iterative Search", - "section": "A Three Point Grid", - "text": "A Three Point Grid" + "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#partial-pooling", + "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#partial-pooling", + "title": "Extras - Effect Encodings", + "section": "Partial pooling", + "text": "Partial pooling" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#gaussian-processes-and-optimization", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#gaussian-processes-and-optimization", - "title": "4 - Iterative Search", - "section": "Gaussian Processes and Optimization", - "text": "Gaussian Processes and Optimization\nWe can make a “meta-model” with a small set of historical performance results.\nGaussian Processes (GP) models are a good choice to model performance.\n\nIt is a Bayesian model so we are using Bayesian Optimization (BO).\nFor regression, we can assume that our data are multivariate normal.\nWe also define a covariance function for the variance relationship between data points. A common one is:\n\n\\[\\operatorname{cov}(\\boldsymbol{x}_i, \\boldsymbol{x}_j) = \\exp\\left(-\\frac{1}{2}|\\boldsymbol{x}_i - \\boldsymbol{x}_j|^2\\right) + \\sigma^2_{ij}\\]\n\nGPs are good because\n\nthey are flexible regression models (in the sense that splines are flexible).\nwe need to get mean and variance predictions (and they are Bayesian)\ntheir variability is based on spatial distances.\n\nSome people use random forests (with conformal variance estimates) or other methods but GPs are most popular." + "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#agent-effects", + "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#agent-effects", + "title": "Extras - Effect Encodings", + "section": "Agent effects ", + "text": "Agent effects \n\nlibrary(embed)\n\nhotel_effect_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_lencode_mixed(agent, company, outcome = vars(avg_price_per_room)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#predicting-candidates", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#predicting-candidates", - "title": "4 - Iterative Search", - "section": "Predicting Candidates", - "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. MAE)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." + "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#effect-encoding-results", + "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#effect-encoding-results", + "title": "Extras - Effect Encodings", + "section": "Effect encoding results ", + "text": "Effect encoding results \n\nhotel_effect_wflow <-\n workflow() %>%\n add_model(linear_reg()) %>% \n update_recipe(hotel_effect_rec)\n\nreg_metrics <- metric_set(mae, rsq)\n\nhotel_effect_res <-\n hotel_effect_wflow %>%\n fit_resamples(hotel_rs, metrics = reg_metrics)\n\ncollect_metrics(hotel_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.8 10 0.189 Preprocessor1_Model1\n#> 2 rsq standard 0.870 10 0.00357 Preprocessor1_Model1\n\nSlightly worse but it can handle new agents (if they occur)." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#your-turn", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#your-turn", - "title": "4 - Iterative Search", + "objectID": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#your-turn", + "href": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#your-turn", + "title": "6 - Wrapping up", "section": "Your turn", - "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize MAE.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", - "title": "4 - Iterative Search", - "section": "GP Fit (ribbon is mean +/- 1SD)", - "text": "GP Fit (ribbon is mean +/- 1SD)" + "objectID": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#resources-to-keep-learning", + "href": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#resources-to-keep-learning", + "title": "6 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#choosing-new-candidates", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#choosing-new-candidates", - "title": "4 - Iterative Search", - "section": "Choosing New Candidates", - "text": "Choosing New Candidates\nThis isn’t a very good fit but we can still use it.\nHow can we use the outputs to choose the next point to measure?\n\nAcquisition functions take the predicted mean and variance and use them to balance:\n\nexploration: new candidates should explore new areas.\nexploitation: new candidates must stay near existing values.\n\nExploration focuses on the variance, exploitation is about the mean." + "objectID": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#feedback", + "href": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#feedback", + "title": "6 - Wrapping up", + "section": "Feedback", + "text": "Feedback\n\nPlease go to http://pos.it/conf-workshop-survey. Your feedback is crucial!\nData from the survey informs curriculum and format decisions for future conf workshops, and we really appreciate you taking the time to provide it." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#acquisition-functions", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#acquisition-functions", - "title": "4 - Iterative Search", - "section": "Acquisition Functions", - "text": "Acquisition Functions\nWe’ll use an acquisition function to select a new candidate.\nThe most popular method appears to be expected improvement (EI) above the current best results.\n\nZero at existing data points.\nThe expected improvement is integrated over all possible improvement (“expected” in the probability sense).\n\nWe would probably pick the point with the largest EI as the next point.\n(There are other functions beyond EI.)" + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---setup", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---setup", + "title": "3 - Grid Search via Racing", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement", - "title": "4 - Iterative Search", - "section": "Expected Improvement", - "text": "Expected Improvement" + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---data-usage", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---data-usage", + "title": "3 - Grid Search via Racing", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + }, + { + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---boosting-model", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---boosting-model", + "title": "3 - Grid Search via Racing", + "section": "Previously - Boosting Model ", + "text": "Previously - Boosting Model \n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + }, + { + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#making-grid-search-more-efficient", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#making-grid-search-more-efficient", + "title": "3 - Grid Search via Racing", + "section": "Making Grid Search More Efficient", + "text": "Making Grid Search More Efficient\nIn the last section, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\nAlso, for some models, we can fit far fewer models than the number that are being evaluated.\n\nFor boosting, a model with X trees can often predict on candidates with less than X trees.\n\nBoth of these methods can lead to enormous speed-ups." + }, + { + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#model-racing", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#model-racing", + "title": "3 - Grid Search via Racing", + "section": "Model Racing", + "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models but only for a few resamples.\nDetermine which candidates have a low probability of being selected.\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain)\n\nThis can result in fitting a small number of models." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#iteration", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#iteration", - "title": "4 - Iterative Search", - "section": "Iteration", - "text": "Iteration\nOnce we pick the candidate point, we measure performance for it (e.g. resampling).\n\nAnother GP is fit, EI is recomputed, and so on.\n\nWe stop when we have completed the allowed number of iterations or if we don’t see any improvement after a pre-set number of attempts." + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#discarding-candidates", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#discarding-candidates", + "title": "3 - Grid Search via Racing", + "section": "Discarding Candidates", + "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large difference between resamples in the results." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-fit-with-four-points", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-fit-with-four-points", - "title": "4 - Iterative Search", - "section": "GP Fit with four points", - "text": "GP Fit with four points" + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#resampling-results-non-racing", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#resampling-results-non-racing", + "title": "3 - Grid Search via Racing", + "section": "Resampling Results (Non-Racing)", + "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement-1", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement-1", - "title": "4 - Iterative Search", - "section": "Expected Improvement", - "text": "Expected Improvement" + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#are-candidates-different", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#are-candidates-different", + "title": "3 - Grid Search via Racing", + "section": "Are Candidates Different?", + "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval is (0.99, 2.8), indicating that candidate number 2 has smaller error.\nWhat if we were to compare each model candidate to the current best at each resample?\nOne shows superiority when 4 resamples have been evaluated." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-evolution", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-evolution", - "title": "4 - Iterative Search", - "section": "GP Evolution", - "text": "GP Evolution" + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#evaluating-differences-in-candidates", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#evaluating-differences-in-candidates", + "title": "3 - Grid Search via Racing", + "section": "Evaluating Differences in Candidates", + "text": "Evaluating Differences in Candidates" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement-evolution", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement-evolution", - "title": "4 - Iterative Search", - "section": "Expected Improvement Evolution", - "text": "Expected Improvement Evolution" + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#interim-analysis-of-results", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#interim-analysis-of-results", + "title": "3 - Grid Search via Racing", + "section": "Interim Analysis of Results", + "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order.\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#bo-in-tidymodels", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#bo-in-tidymodels", - "title": "4 - Iterative Search", - "section": "BO in tidymodels", - "text": "BO in tidymodels\nWe’ll use a function called tune_bayes() that has very similar syntax to tune_grid().\n\nIt has an additional initial argument for the initial set of performance estimates and parameter combinations for the GP model." + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#racing", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#racing", + "title": "3 - Grid Search via Racing", + "section": "Racing ", + "text": "Racing \n\n# Let's use a larger grid\nset.seed(8945)\nlgbm_grid <- \n lgbm_param %>% \n grid_latin_hypercube(size = 50)\n\nlibrary(finetune)\n\nset.seed(9)\nlgbm_race_res <-\n lgbm_wflow %>%\n tune_race_anova(\n resamples = hotel_rs,\n grid = lgbm_grid, \n metrics = reg_metrics\n )\n\nThe syntax and helper functions are extremely similar to those shown for tune_grid()." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#initial-grid-points", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#initial-grid-points", - "title": "4 - Iterative Search", - "section": "Initial grid points", - "text": "Initial grid points\ninitial can be the results of another tune_*() function or an integer (in which case tune_grid() is used under to hood to make such an initial set of results).\n\nWe’ll run the optimization more than once, so let’s make an initial grid of results to serve as the substrate for the BO.\nI suggest at least the number of tuning parameters plus two as the initial grid for BO." + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#racing-results", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#racing-results", + "title": "3 - Grid Search via Racing", + "section": "Racing Results ", + "text": "Racing Results \n\nshow_best(lgbm_race_res, metric = \"mae\")\n#> # A tibble: 2 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1516 7 0.0421 176 12 mae standard 9.60 10 0.181 Preprocessor42_Model1\n#> 2 1014 5 0.0791 35 181 mae standard 9.61 10 0.179 Preprocessor06_Model1" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#an-initial-grid", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#an-initial-grid", - "title": "4 - Iterative Search", - "section": "An Initial Grid", - "text": "An Initial Grid\n\nreg_metrics <- metric_set(mae, rsq)\n\nset.seed(12)\ninit_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = nrow(lgbm_param) + 2,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(init_res, metric = \"mae\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 390 10 0.0139 13 62 mae standard 11.3 10 0.202 Preprocessor1_Model1\n#> 2 718 31 0.00112 72 25 mae standard 29.0 10 0.335 Preprocessor4_Model1\n#> 3 1236 22 0.0000261 11 17 mae standard 51.8 10 0.416 Preprocessor7_Model1\n#> 4 1044 25 0.00000832 34 12 mae standard 52.8 10 0.424 Preprocessor5_Model1\n#> 5 1599 7 0.0000000402 254 179 mae standard 53.2 10 0.427 Preprocessor6_Model1" + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#racing-results-1", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#racing-results-1", + "title": "3 - Grid Search via Racing", + "section": "Racing Results ", + "text": "Racing Results \n\n\nOnly 171 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_race_res) + \n scale_x_continuous(breaks = pretty_breaks())" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#bo-using-tidymodels", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#bo-using-tidymodels", - "title": "4 - Iterative Search", - "section": "BO using tidymodels", - "text": "BO using tidymodels\n\nset.seed(15)\nlgbm_bayes_res <-\n lgbm_wflow %>%\n tune_bayes(\n resamples = hotel_rs,\n initial = init_res, # <- initial results\n iter = 20,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(lgbm_bayes_res, metric = \"mae\")\n#> # A tibble: 5 × 12\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config .iter\n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> <int>\n#> 1 1769 2 0.0299 114 245 mae standard 9.41 10 0.160 Iter13 13\n#> 2 1969 3 0.0270 240 99 mae standard 9.49 10 0.189 Iter11 11\n#> 3 1780 5 0.0403 27 78 mae standard 9.54 10 0.164 Iter17 17\n#> 4 1454 3 0.0414 114 10 mae standard 9.55 10 0.144 Iter10 10\n#> 5 1253 2 0.0312 131 207 mae standard 9.56 10 0.145 Iter19 19" + "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#your-turn", + "href": "archive/2023-09-posit-conf/advanced-04-racing.html#your-turn", + "title": "3 - Grid Search via Racing", + "section": "Your turn", + "text": "Your turn\n\nRun tune_race_anova() with a different seed.\nDid you get the same or similar results?\n\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results", - "title": "4 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\")" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#working-with-our-predictors", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#working-with-our-predictors", + "title": "1 - Feature Engineering", + "section": "Working with our predictors", + "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for linear regression).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results-1", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results-1", - "title": "4 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"parameters\")" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#what-is-feature-engineering", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#what-is-feature-engineering", + "title": "1 - Feature Engineering", + "section": "What is feature engineering?", + "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPrincipal component analysis (PCA) feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection (FES)." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results-2", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results-2", - "title": "4 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\")" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#example-dates", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#example-dates", + "title": "1 - Feature Engineering", + "section": "Example: Dates", + "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#enhance", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#enhance", - "title": "4 - Iterative Search", - "section": "ENHANCE", - "text": "ENHANCE\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\") +\n ylim(c(9, 14))" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#general-definitions", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#general-definitions", + "title": "1 - Feature Engineering", + "section": "General definitions ", + "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#your-turn-1", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#your-turn-1", - "title": "4 - Iterative Search", - "section": "Your turn", - "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\n\n\n\n−+\n10:00" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#previously---setup", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#previously---setup", + "title": "1 - Feature Engineering", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\n\n# Add another package:\nlibrary(textrecipes)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#notes", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#notes", - "title": "4 - Iterative Search", - "section": "Notes", - "text": "Notes\n\nStopping tune_bayes() will return the current results.\nParallel processing can still be used to more efficiently measure each candidate point.\nThere are a lot of other iterative methods that you can use.\nThe finetune package also has functions for simulated annealing search." + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#previously---data-usage", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#previously---data-usage", + "title": "1 - Feature Engineering", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\n\nWe’ll go from here and create a set of resamples to use for model assessments." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#finalizing-the-model", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#finalizing-the-model", - "title": "4 - Iterative Search", - "section": "Finalizing the Model", - "text": "Finalizing the Model\nLet’s say that we’ve tried a lot of different models and we like our lightgbm model the most.\nWhat do we do now?\n\nFinalize the workflow by choosing the values for the tuning parameters.\nFit the model on the entire training set.\nVerify performance using the test set.\nDocument and publish the model(?)" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#resampling-strategy", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#resampling-strategy", + "title": "1 - Feature Engineering", + "section": "Resampling Strategy", + "text": "Resampling Strategy" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#locking-down-the-tuning-parameters", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#locking-down-the-tuning-parameters", - "title": "4 - Iterative Search", - "section": "Locking Down the Tuning Parameters", - "text": "Locking Down the Tuning Parameters\nWe can take the results of the Bayesian optimization and accept the best results:\n\nbest_param <- select_best(lgbm_bayes_res, metric = \"mae\")\nfinal_wflow <- \n lgbm_wflow %>% \n finalize_workflow(best_param)\nfinal_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: boost_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 4 Recipe Steps\n#> \n#> • step_YeoJohnson()\n#> • step_dummy_hash()\n#> • step_dummy_hash()\n#> • step_zv()\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Boosted Tree Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1769\n#> min_n = 2\n#> learn_rate = 0.0299391312149257\n#> \n#> Engine-Specific Arguments:\n#> num_threads = 1\n#> \n#> Computational engine: lightgbm" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#resampling-strategy-1", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#resampling-strategy-1", + "title": "1 - Feature Engineering", + "section": "Resampling Strategy ", + "text": "Resampling Strategy \nWe’ll use simple 10-fold cross-validation (stratified sampling):\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)\nhotel_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [3372/377]> Fold01\n#> 2 <split [3373/376]> Fold02\n#> 3 <split [3373/376]> Fold03\n#> 4 <split [3373/376]> Fold04\n#> 5 <split [3373/376]> Fold05\n#> 6 <split [3374/375]> Fold06\n#> 7 <split [3375/374]> Fold07\n#> 8 <split [3376/373]> Fold08\n#> 9 <split [3376/373]> Fold09\n#> 10 <split [3376/373]> Fold10" }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#the-final-fit", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#the-final-fit", - "title": "4 - Iterative Search", - "section": "The Final Fit", - "text": "The Final Fit\nWe can use individual functions:\nfinal_fit <- final_wflow %>% fit(data = hotel_train)\n\n# then predict() or augment() \n# then compute metrics\n\nRemember that there is also a convenience function to do all of this:\n\nset.seed(3893)\nfinal_res <- final_wflow %>% last_fit(hotel_split, metrics = reg_metrics)\nfinal_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [3749/1251]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [1,251 × 4]> <workflow>" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", + "title": "1 - Feature Engineering", + "section": "Prepare your data for modeling ", + "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of preprocessing and feature engineering steps.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." }, { - "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#test-set-results", - "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#test-set-results", - "title": "4 - Iterative Search", - "section": "Test Set Results", - "text": "Test Set Results\n\n\n\nfinal_res %>% \n collect_predictions() %>% \n cal_plot_regression(\n truth = avg_price_per_room, \n estimate = .pred, \n alpha = 1 / 4)\n\nTest set performance:\n\nfinal_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 9.63 Preprocessor1_Model1\n#> 2 rsq standard 0.948 Preprocessor1_Model1" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#a-first-recipe", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#a-first-recipe", + "title": "1 - Feature Engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---setup", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---setup", - "title": "2 - Tuning Hyperparameters", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#a-first-recipe-1", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#a-first-recipe-1", + "title": "1 - Feature Engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nsummary(hotel_rec)\n#> # A tibble: 27 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 lead_time <chr [2]> predictor original\n#> 2 stays_in_weekend_nights <chr [2]> predictor original\n#> 3 stays_in_week_nights <chr [2]> predictor original\n#> 4 adults <chr [2]> predictor original\n#> 5 children <chr [2]> predictor original\n#> 6 babies <chr [2]> predictor original\n#> 7 meal <chr [3]> predictor original\n#> 8 country <chr [3]> predictor original\n#> 9 market_segment <chr [3]> predictor original\n#> 10 distribution_channel <chr [3]> predictor original\n#> # ℹ 17 more rows\n\nThe type column contains information on the variables" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---data-usage", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---data-usage", - "title": "2 - Tuning Hyperparameters", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn", + "title": "1 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\nWhat do you think are in the type vectors for the lead_time and country columns?\n\n\n\n−+\n02:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", - "title": "2 - Tuning Hyperparameters", - "section": "Previously - Feature engineering ", - "text": "Previously - Feature engineering \n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#create-indicator-variables", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#create-indicator-variables", + "title": "1 - Feature Engineering", + "section": "Create indicator variables ", + "text": "Create indicator variables \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns.\nstep_dummy() records the levels of the categorical predictors in the training set." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-parameters", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#filter-out-constant-columns", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#filter-out-constant-columns", + "title": "1 - Feature Engineering", + "section": "Filter out constant columns ", + "text": "Filter out constant columns \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#normalization", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#normalization", + "title": "1 - Feature Engineering", + "section": "Normalization ", + "text": "Normalization \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "title": "2 - Tuning Hyperparameters", - "section": "Tagging parameters for tuning ", - "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#reduce-correlation", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#reduce-correlation", + "title": "1 - Feature Engineering", + "section": "Reduce correlation ", + "text": "Reduce correlation \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", - "title": "2 - Tuning Hyperparameters", - "section": "Optimizing the hash features ", - "text": "Optimizing the hash features \nOur new recipe is:\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\n\nWe will be using a tree-based model in a minute.\n\nThe other categorical predictors are left as-is.\nThat’s why there is no step_dummy()." + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps", + "title": "1 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-trees", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-trees", - "title": "2 - Tuning Hyperparameters", - "section": "Boosted Trees", - "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved and new samples are predicted using a weighted average of the votes of each tree in the ensemble.\n\nWe’ll focus on the popular lightgbm implementation." + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-1", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-1", + "title": "1 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = vars(avg_price_per_room))\n\n\nA fancy machine learning supervised dimension reduction technique…\n\nNote that this uses the outcome, and it is from an extension package" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands)\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-2", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-2", + "title": "1 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "title": "2 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nTBH it is usually not difficult to optimize these models.\n\nOften, there are multiple candidate tuning parameter combinations that have very good results.\n\nTo demonstrate simple concepts, we’ll look at optimizing the number of trees in the ensemble (between 1 and 100) and the learning rate (\\(10^{-5}\\) to \\(10^{-1}\\))." + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn-1", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn-1", + "title": "1 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\n\nCreate a recipe() for the hotel data to:\n\nuse a Yeo-Johnson (YJ) transformation on lead_time\nconvert factors to indicator variables\nremove zero-variance variables\nadd the spline technique shown above\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "title": "2 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters ", - "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#minimal-recipe", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#minimal-recipe", + "title": "1 - Feature Engineering", + "section": "Minimal recipe ", + "text": "Minimal recipe \n\nhotel_indicators <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "title": "2 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#measuring-performance", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#measuring-performance", + "title": "1 - Feature Engineering", + "section": "Measuring Performance ", + "text": "Measuring Performance \nWe’ll compute two measures: mean absolute error and the coefficient of determination (a.k.a \\(R^2\\)).\n\\[\\begin{align}\nMAE &= \\frac{1}{n}\\sum_{i=1}^n |y_i - \\hat{y}_i| \\notag \\\\\nR^2 &= cor(y_i, \\hat{y}_i)^2\n\\end{align}\\]\nThe focus will be on MAE for parameter optimization. We’ll use a metric set to compute these:\n\nreg_metrics <- metric_set(mae, rsq)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search", - "title": "2 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#using-a-workflow", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#using-a-workflow", + "title": "1 - Feature Engineering", + "section": "Using a workflow ", + "text": "Using a workflow \n\nset.seed(9)\n\nhotel_lm_wflow <-\n workflow() %>%\n add_recipe(hotel_indicators) %>%\n add_model(linear_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nhotel_lm_res <-\n hotel_lm_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_lm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.6 10 0.214 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00339 Preprocessor1_Model1" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-1", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-1", - "title": "2 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nIn reality we would probably sample the space more densely:" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn-2", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn-2", + "title": "1 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#iterative-search", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#iterative-search", - "title": "2 - Tuning Hyperparameters", - "section": "Iterative Search", - "text": "Iterative Search\nWe could start with a few points and search the space:" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#holdout-predictions", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#holdout-predictions", + "title": "1 - Feature Engineering", + "section": "Holdout predictions ", + "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nlm_cv_pred <- collect_predictions(hotel_lm_res)\nlm_cv_pred %>% print(n = 7)\n#> # A tibble: 3,749 × 5\n#> id .pred .row avg_price_per_room .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 Fold01 75.1 20 40 Preprocessor1_Model1\n#> 2 Fold01 49.3 28 54 Preprocessor1_Model1\n#> 3 Fold01 64.9 45 50 Preprocessor1_Model1\n#> 4 Fold01 52.8 49 42 Preprocessor1_Model1\n#> 5 Fold01 48.6 61 49 Preprocessor1_Model1\n#> 6 Fold01 29.8 66 40 Preprocessor1_Model1\n#> 7 Fold01 36.9 88 49 Preprocessor1_Model1\n#> # ℹ 3,742 more rows" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#parameters", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Parameters", - "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#calibration-plot", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#calibration-plot", + "title": "1 - Feature Engineering", + "section": "Calibration Plot ", + "text": "Calibration Plot \n\nlibrary(probably)\n\ncal_plot_regression(hotel_lm_res, alpha = 1 / 5)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid", - "title": "2 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\nlgbm_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 4 parameters for tuning\n#> \n#> identifier type object\n#> trees trees nparam[+]\n#> learn_rate learn_rate nparam[+]\n#> agent hash num_terms nparam[+]\n#> company hash num_terms nparam[+]\n\n# Individual functions: \ntrees()\n#> # Trees (quantitative)\n#> Range: [1, 2000]\nlearn_rate()\n#> Learning Rate (quantitative)\n#> Transformer: log-10 [1e-100, Inf]\n#> Range (transformed scale): [-10, -1]\n\n\nA parameter set can be updated (e.g. to change the ranges)." + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", + "title": "1 - Feature Engineering", + "section": "What do we do with the agent and company data?", + "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 unique companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor (in the extra materials)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid-1", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid-1", - "title": "2 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\n\n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1629 0.00000440 524 1454\n#> 2 1746 0.0000000751 1009 2865\n#> 3 53 0.0000180 2313 367\n#> 4 442 0.000000445 347 460\n#> 5 1413 0.0000000208 3232 553\n#> 6 1488 0.0000578 3692 639\n#> 7 906 0.000385 602 332\n#> 8 1884 0.00000000101 1127 567\n#> 9 1812 0.0239 961 1183\n#> 10 393 0.000000117 487 1783\n#> # ℹ 15 more rows\n\n\n\n\nA space-filling design tends to perform better than random grids.\nSpace-filling designs are also usually more efficient than regular grids." + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#per-agent-statistics", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#per-agent-statistics", + "title": "1 - Feature Engineering", + "section": "Per-agent statistics", + "text": "Per-agent statistics" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn", - "title": "2 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n−+\n03:00" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#collapsing-factor-levels", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#collapsing-factor-levels", + "title": "1 - Feature Engineering", + "section": "Collapsing factor levels ", + "text": "Collapsing factor levels \nThere is a recipe step that will redefine factor levels based on their frequency in the training set:\n\nhotel_other_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_other(agent, threshold = 0.001) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nUsing this code, 34 agents (out of 98) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", - "title": "2 - Tuning Hyperparameters", - "section": "Create a regular grid ", - "text": "Create a regular grid \n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 4)\n\ngrid\n#> # A tibble: 256 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.0000000001 256 256\n#> 2 667 0.0000000001 256 256\n#> 3 1333 0.0000000001 256 256\n#> 4 2000 0.0000000001 256 256\n#> 5 1 0.0000001 256 256\n#> 6 667 0.0000001 256 256\n#> 7 1333 0.0000001 256 256\n#> 8 2000 0.0000001 256 256\n#> 9 1 0.0001 256 256\n#> 10 667 0.0001 256 256\n#> # ℹ 246 more rows" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#does-othering-help", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#does-othering-help", + "title": "1 - Feature Engineering", + "section": "Does othering help? ", + "text": "Does othering help? \n\nhotel_other_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hotel_other_rec)\n\nhotel_other_res <-\n hotel_other_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.213 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00341 Preprocessor1_Model1\n\nAbout the same MAE and much faster to complete.\nNow let’s look at a more sophisticated tool called effect feature hashing." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-1", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-1", - "title": "2 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\n\nWhat advantage would a regular grid have?" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing", + "title": "1 - Feature Engineering", + "section": "Feature Hashing", + "text": "Feature Hashing\nBetween agent and company, simple dummy variables would create 198 new columns (that are mostly zeros).\nAnother option is to have a binary indicator that combines some levels of these variables.\nFeature hashing (for more see FES, SMLTAR, and TMwR):\n\nuses the character values of the levels\nconverts them to integer hash values\nuses the integers to assign them to a specific indicator column." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", - "title": "2 - Tuning Hyperparameters", - "section": "Update parameter ranges ", - "text": "Update parameter ranges \n\nlgbm_param <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n update(trees = trees(c(1L, 100L)),\n learn_rate = learn_rate(c(-5, -1)))\n\nset.seed(712)\ngrid <- \n lgbm_param %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 75 0.000312 2991 1250\n#> 2 4 0.0000337 899 3088\n#> 3 15 0.0295 520 1578\n#> 4 8 0.0997 1256 3592\n#> 5 80 0.000622 419 258\n#> 6 70 0.000474 2499 1089\n#> 7 35 0.000165 287 2376\n#> 8 64 0.00137 389 359\n#> 9 58 0.0000250 616 881\n#> 10 84 0.0639 2311 2635\n#> # ℹ 15 more rows" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-1", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-1", + "title": "1 - Feature Engineering", + "section": "Feature Hashing", + "text": "Feature Hashing\nSuppose we want to use 32 indicator variables for agent.\nFor a agent with value “Max_Kuhn”, a hashing function converts it to an integer (say 210397726).\nTo assign it to one of the 32 columns, we would use modular arithmetic to assign it to a column:\n\n# For \"Max_Kuhn\" put a '1' in column: \n210397726 %% 32\n#> [1] 30\n\nHash functions are meant to emulate randomness." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#the-results", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#the-results", - "title": "2 - Tuning Hyperparameters", - "section": "The results ", - "text": "The results \n\n\ngrid %>% \n ggplot(aes(trees, learn_rate)) +\n geom_point(size = 4) +\n scale_y_log10()\n\n\n\n\n\n\n\n\n\nNote that the learning rates are uniform on the log-10 scale." + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-pros", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-pros", + "title": "1 - Feature Engineering", + "section": "Feature Hashing Pros", + "text": "Feature Hashing Pros\n\nThe procedure will automatically work on new values of the predictors.\nIt is fast.\n“Signed” hashes add a sign to help avoid aliasing." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Choosing tuning parameters ", - "text": "Choosing tuning parameters \nLet’s take our previous model and tune more parameters:\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)\n\n# Update the feature hash ranges (log-2 units)\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-cons", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-cons", + "title": "1 - Feature Engineering", + "section": "Feature Hashing Cons", + "text": "Feature Hashing Cons\n\nThere is no real logic behind which factor levels are combined.\nWe don’t know how many columns to add (more in the next section).\nSome columns may have all zeros.\nIf a indicator column is important to the model, we can’t easily determine why.\n\n\nThe signed hash make it slightly more possible to differentiate between confounded levels" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-3", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-3", - "title": "2 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE)\n\nlgbm_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = 25,\n # The options below are not required by default\n param_info = lgbm_param, \n control = ctrl,\n metrics = reg_metrics\n )\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes", + "title": "1 - Feature Engineering", + "section": "Feature Hashing in recipes ", + "text": "Feature Hashing in recipes \nThe textrecipes package has a step that can be added to the recipe:\n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nhotel_hash_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hash_rec)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-4", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-4", - "title": "2 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nlgbm_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [3372/377]> Fold01 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,425 × 9]>\n#> 2 <split [3373/376]> Fold02 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 3 <split [3373/376]> Fold03 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 4 <split [3373/376]> Fold04 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 5 <split [3373/376]> Fold05 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 6 <split [3374/375]> Fold06 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,375 × 9]>\n#> 7 <split [3375/374]> Fold07 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,350 × 9]>\n#> 8 <split [3376/373]> Fold08 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>\n#> 9 <split [3376/373]> Fold09 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>\n#> 10 <split [3376/373]> Fold10 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", + "title": "1 - Feature Engineering", + "section": "Feature Hashing in recipes ", + "text": "Feature Hashing in recipes \n\nhotel_hash_res <-\n hotel_hash_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_hash_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.239 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00324 Preprocessor1_Model1\n\nAbout the same performance but now we can handle new values." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-results", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-results", - "title": "2 - Tuning Hyperparameters", - "section": "Grid results ", - "text": "Grid results \n\nautoplot(lgbm_res)" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#debugging-a-recipe", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#debugging-a-recipe", + "title": "1 - Feature Engineering", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. hash_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results", - "title": "2 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res)\n#> # A tibble: 50 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 298 19 4.15e- 9 222 36 mae standard 53.2 10 0.427 Preprocessor01_Model1\n#> 2 298 19 4.15e- 9 222 36 rsq standard 0.810 10 0.00686 Preprocessor01_Model1\n#> 3 1394 5 5.82e- 6 28 21 mae standard 52.9 10 0.424 Preprocessor02_Model1\n#> 4 1394 5 5.82e- 6 28 21 rsq standard 0.810 10 0.00800 Preprocessor02_Model1\n#> 5 774 12 4.41e- 2 27 95 mae standard 9.77 10 0.155 Preprocessor03_Model1\n#> 6 774 12 4.41e- 2 27 95 rsq standard 0.946 10 0.00341 Preprocessor03_Model1\n#> 7 1342 7 6.84e-10 71 17 mae standard 53.2 10 0.427 Preprocessor04_Model1\n#> 8 1342 7 6.84e-10 71 17 rsq standard 0.811 10 0.00785 Preprocessor04_Model1\n#> 9 669 39 8.62e- 7 141 145 mae standard 53.2 10 0.426 Preprocessor05_Model1\n#> 10 669 39 8.62e- 7 141 145 rsq standard 0.807 10 0.00639 Preprocessor05_Model1\n#> # ℹ 40 more rows" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#example", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#example", + "title": "1 - Feature Engineering", + "section": "Example ", + "text": "Example \n\nhash_rec_fit <- prep(hash_rec)\n\n# Get the transformation coefficient\ntidy(hash_rec_fit, number = 1)\n\n# Get the processed data\nbake(hash_rec_fit, hotel_train %>% slice(1:3), contains(\"_agent_\"))" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results-1", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results-1", - "title": "2 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res, summarize = FALSE)\n#> # A tibble: 500 × 10\n#> id trees min_n learn_rate `agent hash` `company hash` .metric .estimator .estimate .config \n#> <chr> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 Fold01 298 19 0.00000000415 222 36 mae standard 51.8 Preprocessor01_Model1\n#> 2 Fold01 298 19 0.00000000415 222 36 rsq standard 0.821 Preprocessor01_Model1\n#> 3 Fold02 298 19 0.00000000415 222 36 mae standard 52.1 Preprocessor01_Model1\n#> 4 Fold02 298 19 0.00000000415 222 36 rsq standard 0.804 Preprocessor01_Model1\n#> 5 Fold03 298 19 0.00000000415 222 36 mae standard 52.2 Preprocessor01_Model1\n#> 6 Fold03 298 19 0.00000000415 222 36 rsq standard 0.786 Preprocessor01_Model1\n#> 7 Fold04 298 19 0.00000000415 222 36 mae standard 51.7 Preprocessor01_Model1\n#> 8 Fold04 298 19 0.00000000415 222 36 rsq standard 0.826 Preprocessor01_Model1\n#> 9 Fold05 298 19 0.00000000415 222 36 mae standard 55.2 Preprocessor01_Model1\n#> 10 Fold05 298 19 0.00000000415 222 36 rsq standard 0.845 Preprocessor01_Model1\n#> # ℹ 490 more rows" + "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#more-on-recipes", + "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#more-on-recipes", + "title": "1 - Feature Engineering", + "section": "More on recipes", + "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", - "title": "2 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \n\nshow_best(lgbm_res, metric = \"rsq\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1890 10 0.0159 115 174 rsq standard 0.948 10 0.00334 Preprocessor12_Model1\n#> 2 774 12 0.0441 27 95 rsq standard 0.946 10 0.00341 Preprocessor03_Model1\n#> 3 1638 36 0.0409 15 120 rsq standard 0.945 10 0.00384 Preprocessor16_Model1\n#> 4 963 23 0.00556 157 13 rsq standard 0.937 10 0.00320 Preprocessor06_Model1\n#> 5 590 5 0.00320 85 73 rsq standard 0.908 10 0.00465 Preprocessor24_Model1" + "objectID": "archive/2023-07-nyr/index.html", + "href": "archive/2023-07-nyr/index.html", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "title": "2 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nlgbm_best <- select_best(lgbm_res, metric = \"mae\")\nlgbm_best\n#> # A tibble: 1 × 6\n#> trees min_n learn_rate `agent hash` `company hash` .config \n#> <int> <int> <dbl> <int> <int> <chr> \n#> 1 1890 10 0.0159 115 174 Preprocessor12_Model1" + "objectID": "archive/2023-07-nyr/index.html#welcome", + "href": "archive/2023-07-nyr/index.html#welcome", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." }, - { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#checking-calibration", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#checking-calibration", - "title": "2 - Tuning Hyperparameters", - "section": "Checking Calibration ", - "text": "Checking Calibration \n\n\nlibrary(probably)\nlgbm_res %>%\n collect_predictions(\n parameters = lgbm_best\n ) %>%\n cal_plot_regression(\n truth = avg_price_per_room,\n estimate = .pred,\n alpha = 1 / 3\n )" + { + "objectID": "archive/2023-07-nyr/index.html#is-this-workshop-for-me", + "href": "archive/2023-07-nyr/index.html#is-this-workshop-for-me", + "title": "Machine learning with tidymodels", + "section": "Is this workshop for me? ", + "text": "Is this workshop for me? \nThis course assumes intermediate R knowledge. This workshop is for you if:\n\nYou can use the magrittr pipe %>% and/or native pipe |>\nYou are familiar with functions from dplyr, tidyr, and ggplot2\nYou can read data into R, transform and reshape data, and make a wide variety of graphs\n\nWe expect participants to have some exposure to basic statistical concepts, but NOT intermediate or expert familiarity with modeling or machine learning." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel", - "title": "2 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" + "objectID": "archive/2023-07-nyr/index.html#preparation", + "href": "archive/2023-07-nyr/index.html#preparation", + "title": "Machine learning with tidymodels", + "section": "Preparation", + "text": "Preparation\nPlease join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2022.02), available at https://www.rstudio.com/download\nThe following R packages, which you can install from the R console:\n\n\n# First, install the pak package: \n\ninstall.packages(\"pak\")\n\n# Then the packages for both days\npkgs <- \n c(\"bonsai\", \"doParallel\", \"finetune\", \"lightgbm\", \"lme4\", \"plumber\", \n \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"stacks\", \"textrecipes\", \n \"tidymodels\", \"tidymodels/modeldatatoo\", \"vetiver\")\npak::pak(pkgs)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", - "title": "2 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." + "objectID": "archive/2023-07-nyr/index.html#slides", + "href": "archive/2023-07-nyr/index.html#slides", + "title": "Machine learning with tidymodels", + "section": "Slides", + "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. There are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to tidymodels\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n\n\n\nAdvanced tidymodels\n\n01: Feature engineering using recipes\n02: Tuning hyperparameters (grid search)\n03: Grid search via racing\n04: Iterative search\n\n\n\nExtra content (time permitting)\n\nTransit Case Study (includes stacking)\nEffect encodings\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "title": "2 - Tuning Hyperparameters", - "section": "Early stopping for boosted trees", - "text": "Early stopping for boosted trees\nWe have directly optimized the number of trees as a tuning parameter.\nInstead we could\n\nSet the number of trees to a single large number.\nStop adding trees when performance gets worse.\n\nThis is known as “early stopping” and there is a parameter for that: stop_iter.\nEarly stopping has a potential to decrease the tuning time." + "objectID": "archive/2023-07-nyr/index.html#code", + "href": "archive/2023-07-nyr/index.html#code", + "title": "Machine learning with tidymodels", + "section": "Code", + "text": "Code\nQuarto files (version 1.4.104) for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-2", - "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-2", - "title": "2 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\n\nSet trees = 2000 and tune the stop_iter parameter.\nNote that you will need to regenerate lgbm_param with your new workflow!\n\n\n\n−+\n10:00" + "objectID": "archive/2023-07-nyr/index.html#past-workshops", + "href": "archive/2023-07-nyr/index.html#past-workshops", + "title": "Machine learning with tidymodels", + "section": "Past workshops", + "text": "Past workshops\n\nJuly 2022 at rstudio::conf()\nAugust 2022 in Reykjavik\nJuly 2023 at the New York R Conference" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#workshop-policies", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#workshop-policies", - "title": "1 - Introduction", - "section": "Workshop policies", - "text": "Workshop policies\n\nPlease do not photograph people wearing red lanyards\nThere are gender-neutral bathrooms located are among the Grand Suite Bathrooms\nThere are two meditation/prayer rooms: Grand Suite 2A and 2B\nA lactation room is located in Grand Suite 1\nThe meditation/prayer and lactation rooms are open\nSun - Tue 7:30am - 7:00pm, Wed 8:00am - 6:00pm" + "objectID": "archive/2023-07-nyr/index.html#acknowledgments", + "href": "archive/2023-07-nyr/index.html#acknowledgments", + "title": "Machine learning with tidymodels", + "section": "Acknowledgments", + "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#workshop-policies-1", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#workshop-policies-1", - "title": "1 - Introduction", - "section": "Workshop policies", - "text": "Workshop policies\n\nPlease review the code of conduct and COVID policies, which apply to all workshops: https://posit.co/code-of-conduct/.\nCoC site has info on how to report a problem (in person, email, phone)" + "objectID": "archive/2023-07-nyr/index.html#reuse-and-licensing", + "href": "archive/2023-07-nyr/index.html#reuse-and-licensing", + "title": "Machine learning with tidymodels", + "section": "Reuse and licensing", + "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#who-are-you", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML\nYou have used some tidymodels packages\nYou have some experience with evaluating statistical models using resampling techniques" + "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#previously---setup", + "href": "archive/2023-07-nyr/extras-effect-encodings.html#previously---setup", + "title": "Extras - Effect Encodings", + "section": "Previously - Setup", + "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#who-are-tidymodels", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#who-are-tidymodels", - "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nIjeamaka Anyene (Day 1) and Edgar Ruiz (Day 2) are TAing!\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#previously---data-usage", + "href": "archive/2023-07-nyr/extras-effect-encodings.html#previously---data-usage", + "title": "Extras - Effect Encodings", + "section": "Previously - Data Usage", + "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#asking-for-help", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#asking-for-help", - "title": "1 - Introduction", - "section": "Asking for help", - "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" + "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", + "href": "archive/2023-07-nyr/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", + "title": "Extras - Effect Encodings", + "section": "What do we do with the agent and company data?", + "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#section-2", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#section-2", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#per-agent-statistics", + "href": "archive/2023-07-nyr/extras-effect-encodings.html#per-agent-statistics", + "title": "Extras - Effect Encodings", + "section": "Per-agent statistics", + "text": "Per-agent statistics" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#tentative-plan-for-this-workshop", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#tentative-plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Tentative plan for this workshop", - "text": "Tentative plan for this workshop\n\nFeature engineering with recipes\nModel optimization by tuning\n\nGrid search\nRacing\nIterative methods\n\nExtras (time permitting)\n\nEffect encodings\nA case study" + "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#what-is-an-effect-encoding", + "href": "archive/2023-07-nyr/extras-effect-encodings.html#what-is-an-effect-encoding", + "title": "Extras - Effect Encodings", + "section": "What is an effect encoding?", + "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <fct> <int>\n#> 1 52.7 cynthia_worsley 1\n#> 2 51.8 carlos_bryant 2\n#> 3 53.8 lance_hitchcock 3\n#> 4 51.8 lance_hitchcock 4\n#> 5 46.8 cynthia_worsley 5\n#> 6 54.7 charles_najera 6\n#> 7 46.8 cynthia_worsley 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <dbl> <int>\n#> 1 52.7 88.5 1\n#> 2 51.8 89.5 2\n#> 3 53.8 79.8 3\n#> 4 51.8 79.8 4\n#> 5 46.8 88.5 5\n#> 6 54.7 109. 6\n#> 7 46.8 88.5 7\n\n\nThe agent column is replaced with an estimate of the ADR." }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#section-3", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#section-3", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free):\nCheck the workshop channel on Discord for the link!" + "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#per-agent-statistics-again", + "href": "archive/2023-07-nyr/extras-effect-encodings.html#per-agent-statistics-again", + "title": "Extras - Effect Encodings", + "section": "Per-agent statistics again", + "text": "Per-agent statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these means use partial pooling.\nPooling borrows strength across agents and shrinks extreme values towards the mean for agents with very few transations\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for agents with fewer reservations by shrinking the estimate to the overall ADR mean" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#lets-install-some-packages", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of RStudio Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\",\n \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\",\n \"splines2\", \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \n \"vetiver\", \"remotes\")\n\ninstall.packages(pkgs)\n\n\n\n Or log in to Posit Cloud\nLink in our Discord channel!" + "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#partial-pooling", + "href": "archive/2023-07-nyr/extras-effect-encodings.html#partial-pooling", + "title": "Extras - Effect Encodings", + "section": "Partial pooling", + "text": "Partial pooling" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#hotel-data", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#hotel-data", - "title": "1 - Introduction", - "section": "Hotel Data ", - "text": "Hotel Data \nWe’ll use data on hotels to predict the cost of a room.\nThe data are in the modeldata package. We’ll sample down the data and refactor some columns:\n\n\n\nlibrary(tidymodels)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#agent-effects", + "href": "archive/2023-07-nyr/extras-effect-encodings.html#agent-effects", + "title": "Extras - Effect Encodings", + "section": "Agent effects ", + "text": "Agent effects \n\nlibrary(embed)\n\nhotel_effect_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_YeoJohnson(lead_time) %>%\n step_lencode_mixed(agent, company, outcome = vars(avg_price_per_room)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#hotel-date-columns", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#hotel-date-columns", - "title": "1 - Introduction", - "section": "Hotel date columns", - "text": "Hotel date columns\n\nnames(hotel_rates)\n#> [1] \"avg_price_per_room\" \"lead_time\" \n#> [3] \"stays_in_weekend_nights\" \"stays_in_week_nights\" \n#> [5] \"adults\" \"children\" \n#> [7] \"babies\" \"meal\" \n#> [9] \"country\" \"market_segment\" \n#> [11] \"distribution_channel\" \"is_repeated_guest\" \n#> [13] \"previous_cancellations\" \"previous_bookings_not_canceled\"\n#> [15] \"reserved_room_type\" \"assigned_room_type\" \n#> [17] \"booking_changes\" \"agent\" \n#> [19] \"company\" \"days_in_waiting_list\" \n#> [21] \"customer_type\" \"required_car_parking_spaces\" \n#> [23] \"total_of_special_requests\" \"arrival_date_num\" \n#> [25] \"near_christmas\" \"near_new_years\" \n#> [27] \"historical_adr\"" + "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#effect-encoding-results", + "href": "archive/2023-07-nyr/extras-effect-encodings.html#effect-encoding-results", + "title": "Extras - Effect Encodings", + "section": "Effect encoding results ", + "text": "Effect encoding results \n\nhotel_effect_wflow <-\n workflow() %>%\n add_model(linear_reg()) %>% \n update_recipe(hotel_effect_rec)\n\nreg_metrics <- metric_set(mae, rsq)\n\nhotel_effect_res <-\n hotel_effect_wflow %>%\n fit_resamples(hotel_rs, metrics = reg_metrics)\n\ncollect_metrics(hotel_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.8 10 0.236 Preprocessor1_Model1\n#> 2 rsq standard 0.867 10 0.00377 Preprocessor1_Model1\n\nSlightly worse but it can handle new agents (if they occur)." }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#data-splitting-strategy", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#data-splitting-strategy", - "title": "1 - Introduction", - "section": "Data splitting strategy", - "text": "Data splitting strategy" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#previously---setup", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#previously---setup", + "title": "4 - Iterative Search", + "section": "Previously - Setup", + "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#data-spending", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#data-spending", - "title": "1 - Introduction", - "section": "Data Spending ", - "text": "Data Spending \nLet’s split the data into a training set (75%) and testing set (25%) using stratification:\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#previously---data-usage", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#previously---data-usage", + "title": "4 - Iterative Search", + "section": "Previously - Data Usage", + "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#your-turn", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#your-turn", - "title": "1 - Introduction", - "section": "Your turn", - "text": "Your turn\nLet’s take some time and investigate the training data. The outcome is avg_price_per_room.\nAre there any interesting characteristics of the data?\n\n\n\n−+\n10:00" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#previously---boosting-model", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#previously---boosting-model", + "title": "4 - Iterative Search", + "section": "Previously - Boosting Model", + "text": "Previously - Boosting Model\n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\")\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" }, { - "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#our-versions", - "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nR version 4.2.2 (2022-10-31), Quarto (1.4.104)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.2.1\n\n\nbroom\n1.0.5\n\n\ndials\n1.2.0\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.3\n\n\nembed\n1.1.2\n\n\nfinetune\n1.1.0\n\n\nggplot2\n3.4.3\n\n\nlightgbm\n3.3.5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nlme4\n1.1-34\n\n\nmodeldata\n1.2.0\n\n\nparsnip\n1.1.1\n\n\nplumber\n1.2.1\n\n\nprobably\n1.0.2\n\n\npurrr\n1.0.2\n\n\nranger\n0.15.1\n\n\nrecipes\n1.0.8\n\n\nremotes\n2.4.2.1\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nrpart\n4.1.19\n\n\nrpart.plot\n3.1.1\n\n\nrsample\n1.2.0\n\n\nrules\n1.0.2\n\n\nscales\n1.2.1\n\n\nsplines2\n0.5.1\n\n\nstacks\n1.0.2\n\n\ntext2vec\n0.6.3\n\n\ntextrecipes\n1.0.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.1.1\n\n\ntidyr\n1.3.0\n\n\ntune\n1.1.2\n\n\nvetiver\n0.2.4\n\n\nworkflows\n1.1.3\n\n\nworkflowsets\n1.0.1\n\n\nyardstick\n1.2.0" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#iterative-search", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#iterative-search", + "title": "4 - Iterative Search", + "section": "Iterative Search", + "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our boosted tree.\n\nWe could do something like:\nmae_pred <- lm(mae ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#chicago-l-train-data", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#chicago-l-train-data", - "title": "Case Study on Transportation", - "section": "Chicago L-Train data", - "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#iterative-search-1", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#iterative-search-1", + "title": "4 - Iterative Search", + "section": "Iterative Search", + "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between MAE and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#predictors", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#predictors", - "title": "Case Study on Transportation", - "section": "Predictors", - "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#a-large-grid", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#a-large-grid", + "title": "4 - Iterative Search", + "section": "A Large Grid", + "text": "A Large Grid" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#l-train-locations", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#l-train-locations", - "title": "Case Study on Transportation", - "section": "L Train Locations", - "text": "L Train Locations" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#a-three-point-grid", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#a-three-point-grid", + "title": "4 - Iterative Search", + "section": "A Three Point Grid", + "text": "A Three Point Grid" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#your-turn-explore-the-data", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#your-turn-explore-the-data", - "title": "Case Study on Transportation", - "section": "Your turn: Explore the Data", - "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#gaussian-processes-and-optimization", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#gaussian-processes-and-optimization", + "title": "4 - Iterative Search", + "section": "Gaussian Processes and Optimization", + "text": "Gaussian Processes and Optimization\nWe can make a “meta-model” with a small set of historical performance results.\nGaussian Processes (GP) models are a good choice to model performance.\n\nIt is a Bayesian model so we are using Bayesian Optimization (BO).\nFor regression, we can assume that our data are multivariate normal.\nWe also define a covariance function for the variance relationship between data points. A common one is:\n\n\\[\\operatorname{cov}(\\boldsymbol{x}_i, \\boldsymbol{x}_j) = \\exp\\left(-\\frac{1}{2}|\\boldsymbol{x}_i - \\boldsymbol{x}_j|^2\\right) + \\sigma^2_{ij}\\]\n\nGPs are good because\n\nthey are flexible regression models (in the sense that splines are flexible).\nwe need to get mean and variance predictions (and they are Bayesian)\ntheir variability is based on spatial distances.\n\nSome people use random forests (with conformal variance estimates) or other methods but GPs are most popular." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#splitting-with-chicago-data", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#splitting-with-chicago-data", - "title": "Case Study on Transportation", - "section": "Splitting with Chicago data ", - "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#predicting-candidates", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#predicting-candidates", + "title": "4 - Iterative Search", + "section": "Predicting Candidates", + "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. MAE)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#time-series-resampling", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#time-series-resampling", - "title": "Case Study on Transportation", - "section": "Time series resampling", - "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#your-turn", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#your-turn", + "title": "4 - Iterative Search", + "section": "Your turn", + "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize MAE.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#rolling-forecast-origin-resampling", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#rolling-forecast-origin-resampling", - "title": "Case Study on Transportation", - "section": "Rolling forecast origin resampling", - "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#gp-fit-ribbon-is-mean---1sd", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#gp-fit-ribbon-is-mean---1sd", + "title": "4 - Iterative Search", + "section": "GP Fit (ribbon is mean +/- 1SD)", + "text": "GP Fit (ribbon is mean +/- 1SD)" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#choosing-new-candidates", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#choosing-new-candidates", + "title": "4 - Iterative Search", + "section": "Choosing New Candidates", + "text": "Choosing New Candidates\nThis isn’t a very good fit but we can still use it.\nHow can we use the outputs to choose the next point to measure?\n\nAcquisition functions take the predicted mean and variance and use them to balance:\n\nexploration: new candidates should explore new areas.\nexploitation: new candidates must stay near existing values.\n\nExploration focuses on the variance, exploitation is about the mean." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-1", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-1", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#acquisition-functions", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#acquisition-functions", + "title": "4 - Iterative Search", + "section": "Acquisition Functions", + "text": "Acquisition Functions\nWe’ll use an acquisition function to select a new candidate.\nThe most popular method appears to be expected improvement (EI) above the current best results.\n\nZero at existing data points.\nThe expected improvement is integrated over all possible improvement (“expected” in the probability sense).\n\nWe would probably pick the point with the largest EI as the next point.\n(There are other functions beyond EI.)" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-2", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-2", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement", + "title": "4 - Iterative Search", + "section": "Expected Improvement", + "text": "Expected Improvement" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-3", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-3", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#iteration", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#iteration", + "title": "4 - Iterative Search", + "section": "Iteration", + "text": "Iteration\nOnce we pick the candidate point, we measure performance for it (e.g. resampling).\n\nAnother GP is fit, EI is recomputed, and so on.\n\nWe stop when we have completed the allowed number of iterations or if we don’t see any improvement after a pre-set number of attempts." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-4", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-4", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#gp-fit-with-four-points", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#gp-fit-with-four-points", + "title": "4 - Iterative Search", + "section": "GP Fit with four points", + "text": "GP Fit with four points" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#our-resampling-object", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#our-resampling-object", - "title": "Case Study on Transportation", - "section": "Our resampling object ", - "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement-1", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement-1", + "title": "4 - Iterative Search", + "section": "Expected Improvement", + "text": "Expected Improvement" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#feature-engineering-with-recipes", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#feature-engineering-with-recipes", - "title": "Case Study on Transportation", - "section": "Feature engineering with recipes ", - "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#gp-evolution", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#gp-evolution", + "title": "4 - Iterative Search", + "section": "GP Evolution", + "text": "GP Evolution" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe", - "title": "Case Study on Transportation", - "section": "A recipe", - "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement-evolution", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement-evolution", + "title": "4 - Iterative Search", + "section": "Expected Improvement Evolution", + "text": "Expected Improvement Evolution" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#bo-in-tidymodels", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#bo-in-tidymodels", + "title": "4 - Iterative Search", + "section": "BO in tidymodels", + "text": "BO in tidymodels\nWe’ll use a function called tune_bayes() that has very similar syntax to tune_grid().\n\nIt has an additional initial argument for the initial set of performance estimates and parameter combinations for the GP model." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates-1", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates-1", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#initial-grid-points", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#initial-grid-points", + "title": "4 - Iterative Search", + "section": "Initial grid points", + "text": "Initial grid points\ninitial can be the results of another tune_*() function or an integer (in which case tune_grid() is used under to hood to make such an initial set of results).\n\nWe’ll run the optimization more than once, so let’s make an initial grid of results to serve as the substrate for the BO.\nI suggest at least the number of tuning parameters plus two as the initial grid for BO." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates-2", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates-2", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#an-initial-grid", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#an-initial-grid", + "title": "4 - Iterative Search", + "section": "An Initial Grid", + "text": "An Initial Grid\n\nreg_metrics <- metric_set(mae, rsq)\n\nset.seed(12)\ninit_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = nrow(lgbm_param) + 2,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(init_res, metric = \"mae\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 390 10 0.0139 13 62 mae standard 11.9 10 0.208 Preprocessor1_Model1\n#> 2 718 31 0.00112 72 25 mae standard 29.1 10 0.325 Preprocessor4_Model1\n#> 3 1236 22 0.0000261 11 17 mae standard 51.8 10 0.416 Preprocessor7_Model1\n#> 4 1044 25 0.00000832 34 12 mae standard 52.8 10 0.424 Preprocessor5_Model1\n#> 5 1599 7 0.0000000402 254 179 mae standard 53.2 10 0.427 Preprocessor6_Model1" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---remove-constant-columns", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---remove-constant-columns", - "title": "Case Study on Transportation", - "section": "A recipe - remove constant columns ", - "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE) %>% \n step_zv(all_nominal_predictors())" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#bo-using-tidymodels", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#bo-using-tidymodels", + "title": "4 - Iterative Search", + "section": "BO using tidymodels", + "text": "BO using tidymodels\n\nset.seed(15)\nlgbm_bayes_res <-\n lgbm_wflow %>%\n tune_bayes(\n resamples = hotel_rs,\n initial = init_res, # <- initial results\n iter = 20,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(lgbm_bayes_res, metric = \"mae\")\n#> # A tibble: 5 × 12\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config .iter\n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> <int>\n#> 1 1665 2 0.0593 12 59 mae standard 10.1 10 0.173 Iter13 13\n#> 2 1179 2 0.0552 161 121 mae standard 10.2 10 0.147 Iter7 7\n#> 3 1609 6 0.0592 186 192 mae standard 10.2 10 0.195 Iter17 17\n#> 4 1352 6 0.0799 217 46 mae standard 10.3 10 0.211 Iter4 4\n#> 5 1647 4 0.0819 12 240 mae standard 10.3 10 0.198 Iter20 20" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---handle-correlations", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---handle-correlations", - "title": "Case Study on Transportation", - "section": "A recipe - handle correlations ", - "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results", + "title": "4 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\")" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#make-some-models", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#make-some-models", - "title": "Case Study on Transportation", - "section": "Make some models ", - "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) %>% set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) %>% \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results-1", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results-1", + "title": "4 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"parameters\")" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#process-them-on-the-resamples", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#process-them-on-the-resamples", - "title": "Case Study on Transportation", - "section": "Process them on the resamples", - "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set %>% \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results-2", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results-2", + "title": "4 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\")" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#how-do-the-results-look", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#how-do-the-results-look", - "title": "Case Study on Transportation", - "section": "How do the results look?", - "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 31 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.798 0.104 16 recipe cubis… 1\n#> 2 pca_cubist Preprocessor3_Model3 mae 0.978 0.110 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor4_Model2 mae 0.983 0.122 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model1 mae 0.991 0.127 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor3_Model2 mae 0.991 0.113 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor2_Model2 mae 1.02 0.118 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor1_Model3 mae 1.05 0.134 16 recipe cubis… 7\n#> 8 basic_cubist Preprocessor1_Model8 mae 1.07 0.115 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model7 mae 1.07 0.112 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model6 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 21 more rows" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#enhance", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#enhance", + "title": "4 - Iterative Search", + "section": "ENHANCE", + "text": "ENHANCE\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\") +\n ylim(c(9.5, 14))" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#plot-the-results", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#plot-the-results", - "title": "Case Study on Transportation", - "section": "Plot the results ", - "text": "Plot the results \n\nautoplot(chi_res)" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#your-turn-1", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#your-turn-1", + "title": "4 - Iterative Search", + "section": "Your turn", + "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#pull-out-specific-results", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#pull-out-specific-results", - "title": "Case Study on Transportation", - "section": "Pull out specific results ", - "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n autoplot()" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#notes", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#notes", + "title": "4 - Iterative Search", + "section": "Notes", + "text": "Notes\n\nStopping tune_bayes() will return the current results.\nParallel processing can still be used to more efficiently measure each candidate point.\nThere are a lot of other iterative methods that you can use.\nThe finetune package also has functions for simulated annealing search." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#finalizing-the-model", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#finalizing-the-model", + "title": "4 - Iterative Search", + "section": "Finalizing the Model", + "text": "Finalizing the Model\nLet’s say that we’ve tried a lot of different models and we like our lightgbm model the most.\nWhat do we do now?\n\nFinalize the workflow by choosing the values for the tuning parameters.\nFit the model on the entire training set.\nVerify performance using the test set.\nDocument and publish the model(?)" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-1", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-1", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#locking-down-the-tuning-parameters", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#locking-down-the-tuning-parameters", + "title": "4 - Iterative Search", + "section": "Locking Down the Tuning Parameters", + "text": "Locking Down the Tuning Parameters\nWe can take the results of the Bayesian optimization and accept the best results:\n\nbest_param <- select_best(lgbm_bayes_res, metric = \"mae\")\nfinal_wflow <- \n lgbm_wflow %>% \n finalize_workflow(best_param)\nfinal_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: boost_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 4 Recipe Steps\n#> \n#> • step_YeoJohnson()\n#> • step_dummy_hash()\n#> • step_dummy_hash()\n#> • step_zv()\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Boosted Tree Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1665\n#> min_n = 2\n#> learn_rate = 0.0592557571004946\n#> \n#> Computational engine: lightgbm" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-2", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-2", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#the-final-fit", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#the-final-fit", + "title": "4 - Iterative Search", + "section": "The Final Fit", + "text": "The Final Fit\nWe can use individual functions:\nfinal_fit <- final_wflow %>% fit(data = hotel_tr)\n\n# then predict() or augment() \n# then compute metrics\n\nRemember that there is also a convenience function to do all of this:\n\nset.seed(3893)\nfinal_res <- final_wflow %>% last_fit(hotel_split, metrics = reg_metrics)\nfinal_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [3749/1251]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [1,251 × 4]> <workflow>" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-3", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-3", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#test-set-results", + "href": "archive/2023-07-nyr/advanced-04-iterative.html#test-set-results", + "title": "4 - Iterative Search", + "section": "Test Set Results", + "text": "Test Set Results\n\n\n\nfinal_res %>% \n collect_predictions() %>% \n cal_plot_regression(\n truth = avg_price_per_room, \n estimate = .pred, \n alpha = 1 / 4)\n\nTest set performance:\n\nfinal_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 10.5 Preprocessor1_Model1\n#> 2 rsq standard 0.937 Preprocessor1_Model1" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-4", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-4", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---setup", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---setup", + "title": "2 - Tuning Hyperparameters", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-5", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-5", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---data-usage", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---data-usage", + "title": "2 - Tuning Hyperparameters", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)" + }, + { + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---feature-engineering", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---feature-engineering", + "title": "2 - Tuning Hyperparameters", + "section": "Previously - Feature engineering ", + "text": "Previously - Feature engineering \n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#building-a-model-stack", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#building-a-model-stack", - "title": "Case Study on Transportation", - "section": "Building a model stack ", - "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-parameters", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#start-the-stack-and-add-members", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#start-the-stack-and-add-members", - "title": "Case Study on Transportation", - "section": "Start the stack and add members ", - "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() %>% \n add_candidates(chi_res)" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimize-tuning-parameters", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimize-tuning-parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#estimate-weights-for-each-candidate", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#estimate-weights-for-each-candidate", - "title": "Case Study on Transportation", - "section": "Estimate weights for each candidate ", - "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack)\n\nchi_stack_res\n#> # A tibble: 5 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.343\n#> 2 pca_cubist_3_2 cubist_rules 0.236\n#> 3 basic_cubist_1_4 cubist_rules 0.189\n#> 4 pca_lm_4_1 linear_reg 0.163\n#> 5 pca_cubist_3_3 cubist_rules 0.109" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tagging-parameters-for-tuning", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tagging-parameters-for-tuning", + "title": "2 - Tuning Hyperparameters", + "section": "Tagging parameters for tuning ", + "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#how-did-it-do", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#how-did-it-do", - "title": "Case Study on Transportation", - "section": "How did it do? ", - "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimizing-the-hash-features", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimizing-the-hash-features", + "title": "2 - Tuning Hyperparameters", + "section": "Optimizing the hash features ", + "text": "Optimizing the hash features \nOur new recipe is:\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\n\nWe will be using a tree-based model in a minute.\n\nThe other categorical predictors are left as-is.\nThat’s why there is no step_dummy()." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#what-does-it-use", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#what-does-it-use", - "title": "Case Study on Transportation", - "section": "What does it use? ", - "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-trees", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-trees", + "title": "2 - Tuning Hyperparameters", + "section": "Boosted Trees", + "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved and new samples are predicted using a weighted average of the votes of each tree in the ensemble.\n\nWe’ll focus on the popular lightgbm implementation." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#fit-the-required-candidate-models", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#fit-the-required-candidate-models", - "title": "Case Study on Transportation", - "section": "Fit the required candidate models", - "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands)\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#the-test-set-best-cubist-model", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#the-test-set-best-cubist-model", - "title": "Case Study on Transportation", - "section": "The test set: best Cubist model ", - "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n select_best()\n\ncubist_res <- \n chi_res %>% \n extract_workflow(\"pca_cubist\") %>% \n finalize_workflow(best_cubist) %>% \n last_fit(split = chi_split, metrics = metric_set(mae))" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", + "title": "2 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nTBH it is usually not difficult to optimize these models.\n\nOften, there are multiple candidate tuning parameter combinations that have very good results.\n\nTo demonstrate simple concepts, we’ll look at optimizing the number of trees in the ensemble (between 1 and 100) and the learning rate (\\(10^{-5}\\) to \\(10^{-1}\\))." }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#the-test-set-stack-ensemble", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#the-test-set-stack-ensemble", - "title": "Case Study on Transportation", - "section": "The test set: stack ensemble", - "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) %>% \n bind_cols(chi_test)" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", + "title": "2 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters ", + "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\")\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#compare-the-results", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#compare-the-results", - "title": "Case Study on Transportation", - "section": "Compare the results ", - "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.670 Preprocessor1_Model1\n\nstack_pred %>% mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.689" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "title": "2 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#plot-the-test-set", - "href": "archive/2023-07-nyr/extras-transit-case-study.html#plot-the-test-set", - "title": "Case Study on Transportation", - "section": "Plot the test set ", - "text": "Plot the test set \n\n\nlibrary(probably)\ncubist_res %>% \n collect_predictions() %>% \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search", + "title": "2 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" }, { - "objectID": "archive/2023-07-nyr/annotations.html#section", - "href": "archive/2023-07-nyr/annotations.html#section", - "title": "Annotations", - "section": "👀", - "text": "👀\nThis page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-1", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-1", + "title": "2 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nIn reality we would probably sample the space more densely:" }, { - "objectID": "archive/2023-07-nyr/annotations.html#the-initial-split", - "href": "archive/2023-07-nyr/annotations.html#the-initial-split", - "title": "Annotations", - "section": "The initial split", - "text": "The initial split\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#iterative-search", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#iterative-search", + "title": "2 - Tuning Hyperparameters", + "section": "Iterative Search", + "text": "Iterative Search\nWe could start with a few points and search the space:" }, { - "objectID": "archive/2023-07-nyr/annotations.html#what-is-wrong-with-this", - "href": "archive/2023-07-nyr/annotations.html#what-is-wrong-with-this", - "title": "Annotations", - "section": "What is wrong with this?", - "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#parameters", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Parameters", + "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" }, { - "objectID": "archive/2023-07-nyr/annotations.html#where-are-the-fitted-models", - "href": "archive/2023-07-nyr/annotations.html#where-are-the-fitted-models", - "title": "Annotations", - "section": "Where are the fitted models?", - "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-grid", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-grid", + "title": "2 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\nlgbm_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 4 parameters for tuning\n#> \n#> identifier type object\n#> trees trees nparam[+]\n#> learn_rate learn_rate nparam[+]\n#> agent hash num_terms nparam[+]\n#> company hash num_terms nparam[+]\n\n# Individual functions: \ntrees()\n#> # Trees (quantitative)\n#> Range: [1, 2000]\nlearn_rate()\n#> Learning Rate (quantitative)\n#> Transformer: log-10 [1e-100, Inf]\n#> Range (transformed scale): [-10, -1]\n\n\nA parameter set can be updated (e.g. to change the ranges)." }, { - "objectID": "archive/2023-07-nyr/annotations.html#validation-set", - "href": "archive/2023-07-nyr/annotations.html#validation-set", - "title": "Annotations", - "section": "Validation set", - "text": "Validation set\nThe upcoming version of the rsample package (1.2.0) will have a new set of functions specific to validation sets. They will allow you to make an initial three-way split and still use a validation set with the tune package." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-grid-1", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-grid-1", + "title": "2 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\n\n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1629 0.00000440 524 1454\n#> 2 1746 0.0000000751 1009 2865\n#> 3 53 0.0000180 2313 367\n#> 4 442 0.000000445 347 460\n#> 5 1413 0.0000000208 3232 553\n#> 6 1488 0.0000578 3692 639\n#> 7 906 0.000385 602 332\n#> 8 1884 0.00000000101 1127 567\n#> 9 1812 0.0239 961 1183\n#> 10 393 0.000000117 487 1783\n#> # ℹ 15 more rows\n\n\n\n\nA space-filling design tends to perform better than random grids.\nSpace-filling designs are also usually more efficient than regular grids." }, { - "objectID": "archive/2023-07-nyr/annotations.html#update-parameter-ranges", - "href": "archive/2023-07-nyr/annotations.html#update-parameter-ranges", - "title": "Annotations", - "section": "Update parameter ranges", - "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_spline_natural(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn", + "title": "2 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-07-nyr/annotations.html#early-stopping-for-boosted-trees", - "href": "archive/2023-07-nyr/annotations.html#early-stopping-for-boosted-trees", - "title": "Annotations", - "section": "Early stopping for boosted trees", - "text": "Early stopping for boosted trees\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the lightgbm package.\nbonsai has its own wrapper around (lightgbm::lgb.train()) called bonsai::train_lightgbm(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the lightgbm entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for lightgbm)." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-regular-grid", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-regular-grid", + "title": "2 - Tuning Hyperparameters", + "section": "Create a regular grid ", + "text": "Create a regular grid \n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 4)\n\ngrid\n#> # A tibble: 256 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.0000000001 256 256\n#> 2 667 0.0000000001 256 256\n#> 3 1333 0.0000000001 256 256\n#> 4 2000 0.0000000001 256 256\n#> 5 1 0.0000001 256 256\n#> 6 667 0.0000001 256 256\n#> 7 1333 0.0000001 256 256\n#> 8 2000 0.0000001 256 256\n#> 9 1 0.0001 256 256\n#> 10 667 0.0001 256 256\n#> # ℹ 246 more rows" }, { - "objectID": "archive/2023-07-nyr/annotations.html#per-agent-statistics", - "href": "archive/2023-07-nyr/annotations.html#per-agent-statistics", - "title": "Annotations", - "section": "Per-agent statistics", - "text": "Per-agent statistics\nThe effect encoding method essentially takes the effect of a variable, like agent, and makes a data column for that effect. In our example, affect of the agent on the ADR is quantified by a model and then added as a data column to be used in the model.\nSuppose agent Max has a single reservation in the data and it had an ADR of €200. If we used a naive estimate for Max’s effect, the model is being told that Max should always produce an effect of €200. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned agent Davis, who has taken 250 reservations with an average ADR of €100. Davis’s mean is more predictive because it is estimated with better data (i.e., more total reservations). Partial pooling leverages the entire data set and can borrow strength from all of the agents. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a agent’s data is of good quality, the partial pooling effect estimate is closer to the raw mean. Max’s data is not great and is “shrunk” towards the center of the overall average. Since there is so little known about Max’s reservation history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nSince this example has a numeric outcome, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn-1", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn-1", + "title": "2 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\n\nWhat advantage would a regular grid have?" }, { - "objectID": "archive/2023-07-nyr/annotations.html#agent-effects", - "href": "archive/2023-07-nyr/annotations.html#agent-effects", - "title": "Annotations", - "section": "Agent effects", - "text": "Agent effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage though." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#update-parameter-ranges", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#update-parameter-ranges", + "title": "2 - Tuning Hyperparameters", + "section": "Update parameter ranges ", + "text": "Update parameter ranges \n\nlgbm_param <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n update(trees = trees(c(1L, 100L)),\n learn_rate = learn_rate(c(-5, -1)))\n\nset.seed(712)\ngrid <- \n lgbm_param %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 75 0.000312 2991 1250\n#> 2 4 0.0000337 899 3088\n#> 3 15 0.0295 520 1578\n#> 4 8 0.0997 1256 3592\n#> 5 80 0.000622 419 258\n#> 6 70 0.000474 2499 1089\n#> 7 35 0.000165 287 2376\n#> 8 64 0.00137 389 359\n#> 9 58 0.0000250 616 881\n#> 10 84 0.0639 2311 2635\n#> # ℹ 15 more rows" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#previously---setup", - "href": "archive/2023-07-nyr/advanced-03-racing.html#previously---setup", - "title": "3 - Grid Search via Racing", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#the-results", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#the-results", + "title": "2 - Tuning Hyperparameters", + "section": "The results ", + "text": "The results \n\n\ngrid %>% \n ggplot(aes(trees, learn_rate)) +\n geom_point(size = 4) +\n scale_y_log10()\n\n\n\n\n\n\n\n\n\nNote that the learning rates are uniform on the log-10 scale." }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#previously---data-usage", - "href": "archive/2023-07-nyr/advanced-03-racing.html#previously---data-usage", - "title": "3 - Grid Search via Racing", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choosing-tuning-parameters", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choosing-tuning-parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Choosing tuning parameters ", + "text": "Choosing tuning parameters \nLet’s take our previous model and tune more parameters:\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\")\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)\n\n# Update the feature hash ranges (log-2 units)\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#previously---boosting-model", - "href": "archive/2023-07-nyr/advanced-03-racing.html#previously---boosting-model", - "title": "3 - Grid Search via Racing", - "section": "Previously - Boosting Model ", - "text": "Previously - Boosting Model \n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\")\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-3", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-3", + "title": "2 - Tuning Hyperparameters", + "section": "Grid Search ", + "text": "Grid Search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE)\n\nlgbm_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = 25,\n # The options below are not required by default\n param_info = lgbm_param, \n control = ctrl,\n metrics = reg_metrics\n )\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#making-grid-search-more-efficient", - "href": "archive/2023-07-nyr/advanced-03-racing.html#making-grid-search-more-efficient", - "title": "3 - Grid Search via Racing", - "section": "Making Grid Search More Efficient", - "text": "Making Grid Search More Efficient\nIn the last section, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\nAlso, for some models, we can fit far fewer models than the number that are being evaluated.\n\nFor boosting, a model with X trees can often predict on candidates with less than X trees.\n\nBoth of these methods can lead to enormous speed-ups." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-4", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-4", + "title": "2 - Tuning Hyperparameters", + "section": "Grid Search ", + "text": "Grid Search \n\nlgbm_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [3372/377]> Fold01 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,425 × 9]>\n#> 2 <split [3373/376]> Fold02 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 3 <split [3373/376]> Fold03 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 4 <split [3373/376]> Fold04 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 5 <split [3373/376]> Fold05 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 6 <split [3374/375]> Fold06 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,375 × 9]>\n#> 7 <split [3375/374]> Fold07 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,350 × 9]>\n#> 8 <split [3376/373]> Fold08 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>\n#> 9 <split [3376/373]> Fold09 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>\n#> 10 <split [3376/373]> Fold10 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#model-racing", - "href": "archive/2023-07-nyr/advanced-03-racing.html#model-racing", - "title": "3 - Grid Search via Racing", - "section": "Model Racing", - "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models but only for a few resamples.\nDetermine which candidates have a low probability of being selected.\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain)\n\nThis can result in fitting a small number of models." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-results", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-results", + "title": "2 - Tuning Hyperparameters", + "section": "Grid results ", + "text": "Grid results \n\nautoplot(lgbm_res)" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#discarding-candidates", - "href": "archive/2023-07-nyr/advanced-03-racing.html#discarding-candidates", - "title": "3 - Grid Search via Racing", - "section": "Discarding Candidates", - "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large difference between resamples in the results." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-results", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-results", + "title": "2 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(lgbm_res)\n#> # A tibble: 50 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 298 19 4.15e- 9 222 36 mae standard 53.2 10 0.427 Preprocessor01_Model1\n#> 2 298 19 4.15e- 9 222 36 rsq standard 0.811 10 0.00785 Preprocessor01_Model1\n#> 3 1394 5 5.82e- 6 28 21 mae standard 52.9 10 0.424 Preprocessor02_Model1\n#> 4 1394 5 5.82e- 6 28 21 rsq standard 0.810 10 0.00857 Preprocessor02_Model1\n#> 5 774 12 4.41e- 2 27 95 mae standard 10.5 10 0.175 Preprocessor03_Model1\n#> 6 774 12 4.41e- 2 27 95 rsq standard 0.939 10 0.00381 Preprocessor03_Model1\n#> 7 1342 7 6.84e-10 71 17 mae standard 53.2 10 0.427 Preprocessor04_Model1\n#> 8 1342 7 6.84e-10 71 17 rsq standard 0.810 10 0.00903 Preprocessor04_Model1\n#> 9 669 39 8.62e- 7 141 145 mae standard 53.2 10 0.426 Preprocessor05_Model1\n#> 10 669 39 8.62e- 7 141 145 rsq standard 0.808 10 0.00661 Preprocessor05_Model1\n#> # ℹ 40 more rows" + }, + { + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-results-1", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-results-1", + "title": "2 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(lgbm_res, summarize = FALSE)\n#> # A tibble: 500 × 10\n#> id trees min_n learn_rate `agent hash` `company hash` .metric .estimator .estimate .config \n#> <chr> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 Fold01 298 19 0.00000000415 222 36 mae standard 51.8 Preprocessor01_Model1\n#> 2 Fold01 298 19 0.00000000415 222 36 rsq standard 0.834 Preprocessor01_Model1\n#> 3 Fold02 298 19 0.00000000415 222 36 mae standard 52.1 Preprocessor01_Model1\n#> 4 Fold02 298 19 0.00000000415 222 36 rsq standard 0.801 Preprocessor01_Model1\n#> 5 Fold03 298 19 0.00000000415 222 36 mae standard 52.2 Preprocessor01_Model1\n#> 6 Fold03 298 19 0.00000000415 222 36 rsq standard 0.784 Preprocessor01_Model1\n#> 7 Fold04 298 19 0.00000000415 222 36 mae standard 51.7 Preprocessor01_Model1\n#> 8 Fold04 298 19 0.00000000415 222 36 rsq standard 0.828 Preprocessor01_Model1\n#> 9 Fold05 298 19 0.00000000415 222 36 mae standard 55.2 Preprocessor01_Model1\n#> 10 Fold05 298 19 0.00000000415 222 36 rsq standard 0.850 Preprocessor01_Model1\n#> # ℹ 490 more rows" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#resampling-results-non-racing", - "href": "archive/2023-07-nyr/advanced-03-racing.html#resampling-results-non-racing", - "title": "3 - Grid Search via Racing", - "section": "Resampling Results (Non-Racing)", - "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choose-a-parameter-combination", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choose-a-parameter-combination", + "title": "2 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \n\nshow_best(lgbm_res, metric = \"rsq\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1890 10 0.0159 115 174 rsq standard 0.940 10 0.00369 Preprocessor12_Model1\n#> 2 774 12 0.0441 27 95 rsq standard 0.939 10 0.00381 Preprocessor03_Model1\n#> 3 1638 36 0.0409 15 120 rsq standard 0.938 10 0.00346 Preprocessor16_Model1\n#> 4 963 23 0.00556 157 13 rsq standard 0.930 10 0.00358 Preprocessor06_Model1\n#> 5 590 5 0.00320 85 73 rsq standard 0.905 10 0.00505 Preprocessor24_Model1" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#are-candidates-different", - "href": "archive/2023-07-nyr/advanced-03-racing.html#are-candidates-different", - "title": "3 - Grid Search via Racing", - "section": "Are Candidates Different?", - "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval is (0.99, 2.8), indicating that candidate number 2 has smaller error.\nWhat if we were to compare each model candidate to the current best at each resample?\nOne shows superiority when 4 resamples have been evaluated." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "title": "2 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nlgbm_best <- select_best(lgbm_res, metric = \"mae\")\nlgbm_best\n#> # A tibble: 1 × 6\n#> trees min_n learn_rate `agent hash` `company hash` .config \n#> <int> <int> <dbl> <int> <int> <chr> \n#> 1 774 12 0.0441 27 95 Preprocessor03_Model1" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#evaluating-differences-in-candidates", - "href": "archive/2023-07-nyr/advanced-03-racing.html#evaluating-differences-in-candidates", - "title": "3 - Grid Search via Racing", - "section": "Evaluating Differences in Candidates", - "text": "Evaluating Differences in Candidates" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#checking-calibration", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#checking-calibration", + "title": "2 - Tuning Hyperparameters", + "section": "Checking Calibration ", + "text": "Checking Calibration \n\n\nlibrary(probably)\nlgbm_res %>%\n collect_predictions(\n parameters = lgbm_best\n ) %>%\n cal_plot_regression(\n truth = avg_price_per_room,\n estimate = .pred,\n alpha = 1 / 3\n )" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#interim-analysis-of-results", - "href": "archive/2023-07-nyr/advanced-03-racing.html#interim-analysis-of-results", - "title": "3 - Grid Search via Racing", - "section": "Interim Analysis of Results", - "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order.\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#running-in-parallel", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#running-in-parallel", + "title": "2 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#racing", - "href": "archive/2023-07-nyr/advanced-03-racing.html#racing", - "title": "3 - Grid Search via Racing", - "section": "Racing ", - "text": "Racing \n\n# Let's use a larger grid\nset.seed(8945)\nlgbm_grid <- \n lgbm_param %>% \n grid_latin_hypercube(size = 50)\n\nlibrary(finetune)\n\nset.seed(9)\nlgbm_race_res <-\n lgbm_wflow %>%\n tune_race_anova(\n resamples = hotel_rs,\n grid = lgbm_grid, \n metrics = reg_metrics\n )\n\nThe syntax and helper functions are extremely similar to those shown for tune_grid()." + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#running-in-parallel-1", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#running-in-parallel-1", + "title": "2 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#racing-results", - "href": "archive/2023-07-nyr/advanced-03-racing.html#racing-results", - "title": "3 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\nshow_best(lgbm_race_res, metric = \"mae\")\n#> # A tibble: 2 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1014 5 0.0791 35 181 mae standard 10.3 10 0.202 Preprocessor06_Model1\n#> 2 1516 7 0.0421 176 12 mae standard 10.4 10 0.200 Preprocessor42_Model1" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#early-stopping-for-boosted-trees", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#early-stopping-for-boosted-trees", + "title": "2 - Tuning Hyperparameters", + "section": "Early stopping for boosted trees", + "text": "Early stopping for boosted trees\nWe have directly optimized the number of trees as a tuning parameter.\nInstead we could\n\nSet the number of trees to a single large number.\nStop adding trees when performance gets worse.\n\nThis is known as “early stopping” and there is a parameter for that: stop_iter.\nEarly stopping has a potential to decrease the tuning time." }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#racing-results-1", - "href": "archive/2023-07-nyr/advanced-03-racing.html#racing-results-1", - "title": "3 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\n\nOnly 170 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_race_res) + \n scale_x_continuous(breaks = pretty_breaks())" + "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn-2", + "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn-2", + "title": "2 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\n\nSet trees = 2000 and tune the stop_iter parameter.\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2023-07-nyr/advanced-03-racing.html#your-turn", - "href": "archive/2023-07-nyr/advanced-03-racing.html#your-turn", - "title": "3 - Grid Search via Racing", + "objectID": "archive/2023-07-nyr/08-wrapping-up.html#your-turn", + "href": "archive/2023-07-nyr/08-wrapping-up.html#your-turn", + "title": "8 - Wrapping up", "section": "Your turn", - "text": "Your turn\n\nRun tune_race_anova() with a different seed.\nDid you get the same or similar results?\n\n\n\n\n−+\n10:00" + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#working-with-our-predictors", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#working-with-our-predictors", - "title": "1 - Feature Engineering", - "section": "Working with our predictors", - "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for linear regression).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." + "objectID": "archive/2023-07-nyr/08-wrapping-up.html#resources-to-keep-learning", + "href": "archive/2023-07-nyr/08-wrapping-up.html#resources-to-keep-learning", + "title": "8 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Twitter and at the tidyverse blog for updates!" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#what-is-feature-engineering", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#what-is-feature-engineering", - "title": "1 - Feature Engineering", - "section": "What is feature engineering?", - "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPrincipal component analysis (PCA) feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection (FES)." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglm for generalized linear model (e.g. logistic regression)\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#example-dates", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#example-dates", - "title": "1 - Feature Engineering", - "section": "Example: Dates", - "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#general-definitions", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#general-definitions", - "title": "1 - Feature Engineering", - "section": "General definitions ", - "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-1", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-1", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#hotel-data", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#hotel-data", - "title": "1 - Feature Engineering", - "section": "Hotel Data ", - "text": "Hotel Data \nWe’ll use data on hotels to predict the cost of a room.\nThe data are in the modeldatatoo package. We’ll sample down the data and refactor some columns:\n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-2", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-2", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#data-splitting-strategy", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#data-splitting-strategy", - "title": "1 - Feature Engineering", - "section": "Data splitting strategy", - "text": "Data splitting strategy" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-3", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-3", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#data-spending", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#data-spending", - "title": "1 - Feature Engineering", - "section": "Data Spending ", - "text": "Data Spending \nLet’s split the data into a training set (75%) and testing set (25%):\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-4", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-4", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn", - "title": "1 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\nLet’s take some time and investigate the training data. The outcome is avg_price_per_room.\nAre there any interesting characteristics of the data?\n\n\n\n−+\n10:00" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-5", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-5", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#resampling-strategy", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#resampling-strategy", - "title": "1 - Feature Engineering", - "section": "Resampling Strategy", - "text": "Resampling Strategy" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-6", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-6", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#resampling-strategy-1", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#resampling-strategy-1", - "title": "1 - Feature Engineering", - "section": "Resampling Strategy ", - "text": "Resampling Strategy \nWe’ll use simple 10-fold cross-validation (stratified sampling):\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)\nhotel_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [3372/377]> Fold01\n#> 2 <split [3373/376]> Fold02\n#> 3 <split [3373/376]> Fold03\n#> 4 <split [3373/376]> Fold04\n#> 5 <split [3373/376]> Fold05\n#> 6 <split [3374/375]> Fold06\n#> 7 <split [3375/374]> Fold07\n#> 8 <split [3376/373]> Fold08\n#> 9 <split [3376/373]> Fold09\n#> 10 <split [3376/373]> Fold10" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-7", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-7", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#prepare-your-data-for-modeling", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#prepare-your-data-for-modeling", - "title": "1 - Feature Engineering", - "section": "Prepare your data for modeling ", - "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of preprocessing and feature engineering steps.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-8", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-8", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#a-first-recipe", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#a-first-recipe", - "title": "1 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-1", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-1", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a different model.\n\n\n\n−+\n05:00\n\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#a-first-recipe-1", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#a-first-recipe-1", - "title": "1 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nsummary(hotel_rec)\n#> # A tibble: 28 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 lead_time <chr [2]> predictor original\n#> 2 arrival_date_day_of_month <chr [2]> predictor original\n#> 3 stays_in_weekend_nights <chr [2]> predictor original\n#> 4 stays_in_week_nights <chr [2]> predictor original\n#> 5 adults <chr [2]> predictor original\n#> 6 children <chr [2]> predictor original\n#> 7 babies <chr [2]> predictor original\n#> 8 meal <chr [3]> predictor original\n#> 9 country <chr [3]> predictor original\n#> 10 market_segment <chr [3]> predictor original\n#> # ℹ 18 more rows\n\nThe type column contains information on the variables" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#models-well-be-using-today", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#models-well-be-using-today", + "title": "3 - What makes a model?", + "section": "Models we’ll be using today", + "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-1", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-1", - "title": "1 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\nWhat do you think are in the type vectors for the lead_time and country columns?\n\n\n\n−+\n02:00" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#create-indicator-variables", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#create-indicator-variables", - "title": "1 - Feature Engineering", - "section": "Create indicator variables ", - "text": "Create indicator variables \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns.\nstep_dummy() records the levels of the categorical predictors in the training set." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression-1", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression-1", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#filter-out-constant-columns", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#filter-out-constant-columns", - "title": "1 - Feature Engineering", - "section": "Filter out constant columns ", - "text": "Filter out constant columns \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression-2", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression-2", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{distance}\\)\n\nFind a sigmoid line that separates the two classes" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#normalization", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#normalization", - "title": "1 - Feature Engineering", - "section": "Normalization ", - "text": "Normalization \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" + }, + { + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees-1", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees-1", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#reduce-correlation", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#reduce-correlation", - "title": "1 - Feature Engineering", - "section": "Reduce correlation ", - "text": "Reduce correlation \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees-2", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees-2", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps", - "title": "1 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "title": "3 - What makes a model?", + "section": "All models are wrong, but some are useful!", + "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps-1", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps-1", - "title": "1 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = avg_price_per_room)\n\n\nA fancy machine learning supervised dimension reduction technique…\n\nNote that this uses the outcome, and it is from an extension package" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "title": "3 - What makes a model?", + "section": "Workflows bind preprocessors and models", + "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps-2", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps-2", - "title": "1 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_spline_natural(year_day, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#what-is-wrong-with-this", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#what-is-wrong-with-this", + "title": "3 - What makes a model?", + "section": "What is wrong with this?", + "text": "What is wrong with this?" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-2", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-2", - "title": "1 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\n\nCreate a recipe() for the hotel data to:\n\nuse a Yeo-Johnson (YJ) transformation on lead_time\nconvert factors to indicator variables\nremove zero-variance variables\n\n\n\n\n−+\n03:00" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#why-a-workflow", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#why-a-workflow", + "title": "3 - What makes a model?", + "section": "Why a workflow()? ", + "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering tomorrow!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#minimal-recipe", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#minimal-recipe", - "title": "1 - Feature Engineering", - "section": "Minimal recipe ", - "text": "Minimal recipe \n\nhotel_indicators <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_YeoJohnson(lead_time) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-1", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-1", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\ntree_spec %>% \n fit(tip ~ ., data = taxi_train) \n#> parsnip model object\n#> \n#> n= 7045 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 7045 2069 yes (0.70631654 0.29368346) \n#> 2) company=Chicago Independents,City Service,Sun Taxi,Taxicab Insurance Agency Llc,other 4328 744 yes (0.82809612 0.17190388) \n#> 4) distance< 4.615 2365 254 yes (0.89260042 0.10739958) *\n#> 5) distance>=4.615 1963 490 yes (0.75038207 0.24961793) \n#> 10) distance>=12.565 1069 81 yes (0.92422825 0.07577175) *\n#> 11) distance< 12.565 894 409 yes (0.54250559 0.45749441) \n#> 22) company=Chicago Independents,Sun Taxi,Taxicab Insurance Agency Llc 278 71 yes (0.74460432 0.25539568) *\n#> 23) company=City Service,other 616 278 no (0.45129870 0.54870130) \n#> 46) distance< 7.205 178 59 yes (0.66853933 0.33146067) *\n#> 47) distance>=7.205 438 159 no (0.36301370 0.63698630) *\n#> 3) company=Flash Cab,Taxi Affiliation Services 2717 1325 yes (0.51232978 0.48767022) \n#> 6) distance< 3.235 1331 391 yes (0.70623591 0.29376409) *\n#> 7) distance>=3.235 1386 452 no (0.32611833 0.67388167) \n#> 14) distance>=12.39 344 90 yes (0.73837209 0.26162791) *\n#> 15) distance< 12.39 1042 198 no (0.19001919 0.80998081) *" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#measuring-performance", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#measuring-performance", - "title": "1 - Feature Engineering", - "section": "Measuring Performance ", - "text": "Measuring Performance \nWe’ll compute two measures: mean absolute error and the coefficient of determination (a.k.a \\(R^2\\)).\n\\[\\begin{align}\nMAE &= \\frac{1}{n}\\sum_{i=1}^n |y_i - \\hat{y}_i| \\notag \\\\\nR^2 &= cor(y_i, \\hat{y}_i)^2\n\\end{align}\\]\nThe focus will be on MAE for parameter optimization. We’ll use a metric set to compute these:\n\nreg_metrics <- metric_set(mae, rsq)" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-2", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-2", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\nworkflow() %>%\n add_formula(tip ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = taxi_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 7045 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 7045 2069 yes (0.70631654 0.29368346) \n#> 2) company=Chicago Independents,City Service,Sun Taxi,Taxicab Insurance Agency Llc,other 4328 744 yes (0.82809612 0.17190388) \n#> 4) distance< 4.615 2365 254 yes (0.89260042 0.10739958) *\n#> 5) distance>=4.615 1963 490 yes (0.75038207 0.24961793) \n#> 10) distance>=12.565 1069 81 yes (0.92422825 0.07577175) *\n#> 11) distance< 12.565 894 409 yes (0.54250559 0.45749441) \n#> 22) company=Chicago Independents,Sun Taxi,Taxicab Insurance Agency Llc 278 71 yes (0.74460432 0.25539568) *\n#> 23) company=City Service,other 616 278 no (0.45129870 0.54870130) \n#> 46) distance< 7.205 178 59 yes (0.66853933 0.33146067) *\n#> 47) distance>=7.205 438 159 no (0.36301370 0.63698630) *\n#> 3) company=Flash Cab,Taxi Affiliation Services 2717 1325 yes (0.51232978 0.48767022) \n#> 6) distance< 3.235 1331 391 yes (0.70623591 0.29376409) *\n#> 7) distance>=3.235 1386 452 no (0.32611833 0.67388167) \n#> 14) distance>=12.39 344 90 yes (0.73837209 0.26162791) *\n#> 15) distance< 12.39 1042 198 no (0.19001919 0.80998081) *" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#using-a-workflow", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#using-a-workflow", - "title": "1 - Feature Engineering", - "section": "Using a workflow ", - "text": "Using a workflow \n\nset.seed(9)\n\nhotel_lm_wflow <-\n workflow() %>%\n add_recipe(hotel_indicators) %>%\n add_model(linear_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nhotel_lm_res <-\n hotel_lm_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_lm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.3 10 0.199 Preprocessor1_Model1\n#> 2 rsq standard 0.874 10 0.00400 Preprocessor1_Model1" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-3", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-3", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\nworkflow(tip ~ ., tree_spec) %>% \n fit(data = taxi_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 7045 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 7045 2069 yes (0.70631654 0.29368346) \n#> 2) company=Chicago Independents,City Service,Sun Taxi,Taxicab Insurance Agency Llc,other 4328 744 yes (0.82809612 0.17190388) \n#> 4) distance< 4.615 2365 254 yes (0.89260042 0.10739958) *\n#> 5) distance>=4.615 1963 490 yes (0.75038207 0.24961793) \n#> 10) distance>=12.565 1069 81 yes (0.92422825 0.07577175) *\n#> 11) distance< 12.565 894 409 yes (0.54250559 0.45749441) \n#> 22) company=Chicago Independents,Sun Taxi,Taxicab Insurance Agency Llc 278 71 yes (0.74460432 0.25539568) *\n#> 23) company=City Service,other 616 278 no (0.45129870 0.54870130) \n#> 46) distance< 7.205 178 59 yes (0.66853933 0.33146067) *\n#> 47) distance>=7.205 438 159 no (0.36301370 0.63698630) *\n#> 3) company=Flash Cab,Taxi Affiliation Services 2717 1325 yes (0.51232978 0.48767022) \n#> 6) distance< 3.235 1331 391 yes (0.70623591 0.29376409) *\n#> 7) distance>=3.235 1386 452 no (0.32611833 0.67388167) \n#> 14) distance>=12.39 344 90 yes (0.73837209 0.26162791) *\n#> 15) distance< 12.39 1042 198 no (0.19001919 0.80998081) *" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-3", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-3", - "title": "1 - Feature Engineering", + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-2", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-2", + "title": "3 - What makes a model?", "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n−+\n05:00" + "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code to make a workflow with your own model of choice.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#holdout-predictions", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#holdout-predictions", - "title": "1 - Feature Engineering", - "section": "Holdout predictions ", - "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nlm_val_pred <- collect_predictions(hotel_lm_res)\nlm_val_pred %>% slice(1:7)\n#> # A tibble: 7 × 5\n#> id .pred .row avg_price_per_room .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 Fold01 62.1 20 40 Preprocessor1_Model1\n#> 2 Fold01 48.0 28 54 Preprocessor1_Model1\n#> 3 Fold01 64.6 45 50 Preprocessor1_Model1\n#> 4 Fold01 45.8 49 42 Preprocessor1_Model1\n#> 5 Fold01 45.8 61 49 Preprocessor1_Model1\n#> 6 Fold01 30.0 66 40 Preprocessor1_Model1\n#> 7 Fold01 38.8 88 49 Preprocessor1_Model1" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#predict-with-your-model", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#predict-with-your-model", + "title": "3 - What makes a model?", + "section": "Predict with your model ", + "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(tip ~ ., tree_spec) %>% \n fit(data = taxi_train)" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#calibration-plot", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#calibration-plot", - "title": "1 - Feature Engineering", - "section": "Calibration Plot ", - "text": "Calibration Plot \n\nlibrary(probably)\n\ncal_plot_regression(hotel_lm_res, alpha = 1 / 5)" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-3", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-3", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = taxi_test)\nWhat do you get?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "title": "1 - Feature Engineering", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 unique companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor (in the extra materials)" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-4", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-4", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = taxi_test)\nWhat do you get?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#per-agent-statistics", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#per-agent-statistics", - "title": "1 - Feature Engineering", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#collapsing-factor-levels", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#collapsing-factor-levels", - "title": "1 - Feature Engineering", - "section": "Collapsing factor levels ", - "text": "Collapsing factor levels \nThere is a recipe step that will redefine factor levels based on their frequency in the training set:\n\nhotel_other_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_YeoJohnson(lead_time) %>%\n step_other(agent, threshold = 0.001) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\nUsing this code, 34 agents (out of 98) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model-1", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model-1", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\nroundint = FALSE is only to quiet a warning" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#does-othering-help", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#does-othering-help", - "title": "1 - Feature Engineering", - "section": "Does othering help? ", - "text": "Does othering help? \n\nhotel_other_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hotel_other_rec)\n\nhotel_other_res <-\n hotel_other_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.4 10 0.205 Preprocessor1_Model1\n#> 2 rsq standard 0.874 10 0.00417 Preprocessor1_Model1\n\nAabout the same MAE and much faster to complete.\nNow let’s look at a more sophisticated tool called effect feature hashing." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model-2", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model-2", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing", - "title": "1 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nBetween agent and company, simple dummy variables would create 198 new columns (that are mostly zeros).\nAnother option is to have a binary indicator that combines some levels of these variables.\nFeature hashing (for more see FES, SMLTAR, and TMwR):\n\nuses the character values of the levels\nconverts them to integer hash values\nuses the integers to assign them to a specific indicator column." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-5", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-5", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nExtract the model engine object from your fitted workflow.\n⚠️ Never predict() with any extracted components!\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-1", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-1", - "title": "1 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nSuppose we want to use 32 indicator variables for agent.\nFor a agent with value “Max_Kuhn”, a hashing function converts it to an integer (say 210397726).\nTo assign it to one of the 32 columns, we would use modular arithmetic to assign it to a column:\n\n# For \"Max_Kuhn\" put a '1' in column: \n210397726 %% 32\n#> [1] 30\n\nHash functions are meant to emulate randomness." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#deploying-a-model", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#deploying-a-model", + "title": "3 - What makes a model?", + "section": "Deploying a model ", + "text": "Deploying a model \nHow do you use your new tree_fit model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"taxi\")\nv\n#> \n#> ── taxi ─ <bundled_workflow> model for deployment \n#> A rpart classification modeling workflow using 6 features\n\nLearn more at https://vetiver.rstudio.com" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-pros", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-pros", - "title": "1 - Feature Engineering", - "section": "Feature Hashing Pros", - "text": "Feature Hashing Pros\n\nThe procedure will automatically work on new values of the predictors.\nIt is fast.\n“Signed” hashes add a sign to help avoid aliasing." + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#deploy-your-model-1", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#deploy-your-model-1", + "title": "3 - What makes a model?", + "section": "Deploy your model ", + "text": "Deploy your model \nHow do you use your new model tree_fit in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 2 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/vetiver\n#> ├──/ping (GET)\n#> └──/predict (POST)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-cons", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-cons", - "title": "1 - Feature Engineering", - "section": "Feature Hashing Cons", - "text": "Feature Hashing Cons\n\nThere is no real logic behind which factor levels are combined.\nWe don’t know how many columns to add (more in the next section).\nSome columns may have all zeros.\nIf a indicator column is important to the model, we can’t easily determine why.\n\n\nThe signed hash make it slightly more possible to differentiate between confounded levels" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-6", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-6", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-in-recipes", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-in-recipes", - "title": "1 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \nThe textrecipes package has a step that can be added to the recipe:\n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\nhotel_hash_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hash_rec)" + "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#the-whole-game---status-update", + "href": "archive/2023-07-nyr/03-what-makes-a-model.html#the-whole-game---status-update", + "title": "3 - What makes a model?", + "section": "The whole game - status update", + "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-in-recipes-1", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-in-recipes-1", - "title": "1 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \n\nhotel_hash_res <-\n hotel_hash_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_hash_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.5 10 0.256 Preprocessor1_Model1\n#> 2 rsq standard 0.872 10 0.00395 Preprocessor1_Model1\n\nAbout the same performance but now we can handle new values." + "objectID": "archive/2023-07-nyr/01-introduction.html#who-are-you", + "href": "archive/2023-07-nyr/01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#debugging-a-recipe", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#debugging-a-recipe", - "title": "1 - Feature Engineering", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. hash_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "archive/2023-07-nyr/01-introduction.html#who-are-tidymodels", + "href": "archive/2023-07-nyr/01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#example", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#example", - "title": "1 - Feature Engineering", - "section": "Example ", - "text": "Example \n\nhash_rec_fit <- prep(hash_rec)\n\n# Get the transformation coefficient\ntidy(hash_rec_fit, number = 1)\n\n# Get the processed data\nbake(hash_rec_fit, hotel_tr %>% slice(1:3), contains(\"_agent_\"))" + "objectID": "archive/2023-07-nyr/01-introduction.html#asking-for-help", + "href": "archive/2023-07-nyr/01-introduction.html#asking-for-help", + "title": "1 - Introduction", + "section": "Asking for help", + "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" }, { - "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#more-on-recipes", - "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#more-on-recipes", - "title": "1 - Feature Engineering", - "section": "More on recipes", - "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." + "objectID": "archive/2023-07-nyr/01-introduction.html#section", + "href": "archive/2023-07-nyr/01-introduction.html#section", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#looking-at-predictions", - "href": "archive/2023-07-nyr/04-evaluating-models.html#looking-at-predictions", - "title": "4 - Evaluating models", - "section": "Looking at predictions", - "text": "Looking at predictions\n\naugment(taxi_fit, new_data = taxi_train) %>%\n relocate(tip, .pred_class, .pred_yes, .pred_no)\n#> # A tibble: 7,045 × 10\n#> tip .pred_class .pred_yes .pred_no distance company local dow month hour\n#> <fct> <fct> <dbl> <dbl> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 no no 0.0625 0.937 5.39 Flash … no Sat Mar 12\n#> 2 no yes 0.924 0.0758 18.4 Sun Ta… no Sat Apr 6\n#> 3 no no 0.391 0.609 5.8 other no Tue Jan 10\n#> 4 no no 0.112 0.888 6.85 Flash … no Fri Apr 8\n#> 5 no no 0.129 0.871 9.5 City S… no Wed Jan 7\n#> 6 no no 0.326 0.674 12 other no Fri Apr 11\n#> 7 no no 0.0917 0.908 8.9 Taxi A… no Mon Feb 14\n#> 8 no yes 0.902 0.0980 1.38 other no Fri Apr 16\n#> 9 no no 0.0917 0.908 9.12 Flash … no Wed Apr 9\n#> 10 no yes 0.933 0.0668 2.28 City S… no Thu Apr 16\n#> # ℹ 7,035 more rows" + "objectID": "archive/2023-07-nyr/01-introduction.html#tentative-plan-for-this-workshop", + "href": "archive/2023-07-nyr/01-introduction.html#tentative-plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Tentative plan for this workshop", + "text": "Tentative plan for this workshop\n\n\n\nToday:\n\nYour data budget\nWhat makes a model\nEvaluating models\n\n\n\n\nTomorrow:\n\nFeature engineering\nTuning hyperparameters\nRacing methods\nIterative search methods" + }, + { + "objectID": "archive/2023-07-nyr/01-introduction.html#section-1", + "href": "archive/2023-07-nyr/01-introduction.html#section-1", + "title": "1 - Introduction", + "section": "", + "text": "Introduce yourself to your neighbors 👋\n\nCheck Slack (#ml-ws-2023) for an RStudio Cloud link." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix", - "href": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix" + "objectID": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning", + "href": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix-1", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(taxi_fit, new_data = taxi_train) %>%\n conf_mat(truth = tip, estimate = .pred_class)\n#> Truth\n#> Prediction yes no\n#> yes 4639 660\n#> no 337 1409" + "objectID": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning-1", + "href": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning-1", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix-2", - "href": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix-2", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(taxi_fit, new_data = taxi_train) %>%\n conf_mat(truth = tip, estimate = .pred_class) %>%\n autoplot(type = \"heatmap\")" + "objectID": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning-2", + "href": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning-2", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance", - "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n accuracy(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858" + "objectID": "archive/2023-07-nyr/01-introduction.html#your-turn", + "href": "archive/2023-07-nyr/01-introduction.html#your-turn", + "title": "1 - Introduction", + "section": "Your turn", + "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-accuracy", - "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-accuracy", - "title": "4 - Evaluating models", - "section": "Dangers of accuracy ", - "text": "Dangers of accuracy \nWe need to be careful of using accuracy() since it can give “good” performance by only predicting one way with imbalanced data\n\naugment(taxi_fit, new_data = taxi_train) %>%\n mutate(.pred_class = factor(\"yes\", levels = c(\"yes\", \"no\"))) %>%\n accuracy(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.706" + "objectID": "archive/2023-07-nyr/01-introduction.html#what-is-tidymodels", + "href": "archive/2023-07-nyr/01-introduction.html#what-is-tidymodels", + "title": "1 - Introduction", + "section": "What is tidymodels? ", + "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.1.0 ──\n#> ✔ broom 1.0.5 ✔ rsample 1.1.1.9000\n#> ✔ dials 1.2.0 ✔ tibble 3.2.1 \n#> ✔ dplyr 1.1.2 ✔ tidyr 1.3.0 \n#> ✔ infer 1.0.4 ✔ tune 1.1.1.9001\n#> ✔ modeldata 1.1.0 ✔ workflows 1.1.3 \n#> ✔ parsnip 1.1.0.9003 ✔ workflowsets 1.0.1 \n#> ✔ purrr 1.0.1 ✔ yardstick 1.2.0.9001\n#> ✔ recipes 1.0.6\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Use tidymodels_prefer() to resolve common conflicts." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-1", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n sensitivity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.932" + "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game", + "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\nPart of any modelling process is\n\nSplitting your data into training and test set\nUsing a resampling scheme\nFitting models\nAssessing performance\nChoosing a model\nFitting and assessing the final model" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-2", - "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-2", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n sensitivity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.932\n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n specificity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.681" + "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-1", + "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-1", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-3", - "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-3", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_fit, new_data = taxi_train) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858\n#> 2 specificity binary 0.681\n#> 3 sensitivity binary 0.932" + "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-2", + "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-2", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-4", - "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-4", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_fit, new_data = taxi_train) %>%\n group_by(local) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 6 × 4\n#> local .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 yes accuracy binary 0.840\n#> 2 no accuracy binary 0.862\n#> 3 yes specificity binary 0.346\n#> 4 no specificity binary 0.719\n#> 5 yes sensitivity binary 0.969\n#> 6 no sensitivity binary 0.925" + "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-3", + "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-3", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#two-class-data", - "href": "archive/2023-07-nyr/04-evaluating-models.html#two-class-data", - "title": "4 - Evaluating models", - "section": "Two class data", - "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" + "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-4", + "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-4", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#varying-the-threshold", - "href": "archive/2023-07-nyr/04-evaluating-models.html#varying-the-threshold", - "title": "4 - Evaluating models", - "section": "Varying the threshold", - "text": "Varying the threshold" + "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-5", + "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-5", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#roc-curves", - "href": "archive/2023-07-nyr/04-evaluating-models.html#roc-curves", - "title": "4 - Evaluating models", - "section": "ROC curves", - "text": "ROC curves\nTo make an ROC (receiver operator characteristic) curve, we:\n\ncalculate the sensitivity and specificity for all possible thresholds\nplot false positive rate (x-axis) versus true positive rate (y-axis)\n\ngiven that sensitivity is the true positive rate, and specificity is the true negative rate. Hence 1 - specificity is the false positive rate.\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." + "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-6", + "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-6", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#roc-curves-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#roc-curves-1", - "title": "4 - Evaluating models", - "section": "ROC curves ", - "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_curve(truth = tip, .pred_yes) %>%\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.25 0.486 0.972\n#> 3 0.6 0.705 0.920\n\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_auc(truth = tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.868" + "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-7", + "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-7", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#roc-curve-plot", - "href": "archive/2023-07-nyr/04-evaluating-models.html#roc-curve-plot", - "title": "4 - Evaluating models", - "section": "ROC curve plot ", - "text": "ROC curve plot \n\n\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_curve(truth = tip, .pred_yes) %>%\n autoplot()" + "objectID": "archive/2023-07-nyr/01-introduction.html#lets-install-some-packages", + "href": "archive/2023-07-nyr/01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of RStudio Cloud:\n\ninstall.packages(\"pak\")\n\npkgs <- c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\", \n \"parallelly\", \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \n \"stacks\", \"textrecipes\", \"tidymodels\", \"tidymodels/modeldatatoo\", \n \"vetiver\")\npak::pak(pkgs)\n\n\nCheck Slack (#ml-ws-2023) for an RStudio Cloud link." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn", - "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/01-introduction.html#our-versions", + "href": "archive/2023-07-nyr/01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nbonsai (0.2.1.9000, Github (tidymodels/bonsai@aab79), broom (1.0.5, local), dials (1.2.0, CRAN), doParallel (1.0.17, CRAN), dplyr (1.1.2, CRAN), embed (1.0.0, CRAN), finetune (1.1.0.9000, Github (tidymodels/finetune@52d), ggplot2 (3.4.2, CRAN), lightgbm (3.3.5, CRAN), lme4 (1.1-33, CRAN), modeldata (1.1.0, CRAN), modeldatatoo (0.1.0.9000, Github (tidymodels/modeldatatoo), parallelly (1.36.0, CRAN), parsnip (1.1.0.9003, Github (tidymodels/parsnip@e627), plumber (1.2.1, CRAN), probably (1.0.2, CRAN), purrr (1.0.1, CRAN), ranger (0.15.1, CRAN), recipes (1.0.6, CRAN), rpart (4.1.19, CRAN), rpart.plot (3.1.1, CRAN), rsample (1.1.1.9000, Github (tidymodels/rsample@afc4), scales (1.2.1, CRAN), stacks (1.0.2.9000, local), textrecipes (1.0.2, CRAN), tibble (3.2.1, CRAN), tidymodels (1.1.0, CRAN), tidyr (1.3.0, CRAN), tune (1.1.1.9001, Github (tidymodels/tune@fea8b02), vetiver (0.2.0, CRAN), workflows (1.1.3, CRAN), workflowsets (1.0.1, CRAN), yardstick (1.2.0.9001, Github (tidymodels/yardstick@6c), and Quarto (1.3.433)" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-1", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html", + "href": "archive/2022-08-Reykjavik-City/annotations.html", + "title": "Annotations", + "section": "", + "text": "This page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources.\n\n\n\nThis is a pretty complex data usage scheme. That is mostly because of the validation set. In every other case, the situation is much more simple.\nThe important point here is that: tidymodels does most of this work for you. In other words, you don’t have to directly specify which data are being used where.\nIn a later section, we will talk about methods of resampling. These methods are like repeated validation sets. As an example, the popular 10-fold cross-validation method is one such type of resampling. Validation sets are special cases of resampling where there is a single “resample”.\nMost types of resampling use multiple hold-out sets of samples from the training set. In those cases, a diagram for data usage here would look like\n\n\n\n\n\n\n\n\n\nIn this case there is just “testing” and “training”. Once the final model is determined, the entire training set is used for the last fit.\nThis is the process that will be used for the tree frog data." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-2", - "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-2", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#section", + "href": "archive/2022-08-Reykjavik-City/annotations.html#section", + "title": "Annotations", + "section": "", + "text": "This page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-3", - "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-3", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\ntaxi_fit %>%\n augment(taxi_train)\n#> # A tibble: 7,045 × 10\n#> tip distance company local dow month hour .pred_class .pred_yes .pred_no\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int> <fct> <dbl> <dbl>\n#> 1 no 5.39 Flash … no Sat Mar 12 no 0.0625 0.937 \n#> 2 no 18.4 Sun Ta… no Sat Apr 6 yes 0.924 0.0758\n#> 3 no 5.8 other no Tue Jan 10 no 0.391 0.609 \n#> 4 no 6.85 Flash … no Fri Apr 8 no 0.112 0.888 \n#> 5 no 9.5 City S… no Wed Jan 7 no 0.129 0.871 \n#> 6 no 12 other no Fri Apr 11 no 0.326 0.674 \n#> 7 no 8.9 Taxi A… no Mon Feb 14 no 0.0917 0.908 \n#> 8 no 1.38 other no Fri Apr 16 yes 0.902 0.0980\n#> 9 no 9.12 Flash … no Wed Apr 9 no 0.0917 0.908 \n#> 10 no 2.28 City S… no Thu Apr 16 yes 0.933 0.0668\n#> # ℹ 7,035 more rows\n\nWe call this “resubstitution” or “repredicting the training set”" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#finalize-and-verify", + "href": "archive/2022-08-Reykjavik-City/annotations.html#finalize-and-verify", + "title": "Annotations", + "section": "", + "text": "This is a pretty complex data usage scheme. That is mostly because of the validation set. In every other case, the situation is much more simple.\nThe important point here is that: tidymodels does most of this work for you. In other words, you don’t have to directly specify which data are being used where.\nIn a later section, we will talk about methods of resampling. These methods are like repeated validation sets. As an example, the popular 10-fold cross-validation method is one such type of resampling. Validation sets are special cases of resampling where there is a single “resample”.\nMost types of resampling use multiple hold-out sets of samples from the training set. In those cases, a diagram for data usage here would look like\n\n\n\n\n\n\n\n\n\nIn this case there is just “testing” and “training”. Once the final model is determined, the entire training set is used for the last fit.\nThis is the process that will be used for the tree frog data." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-4", - "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-4", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858\n\nWe call this a “resubstitution estimate”" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#data-splitting-and-spending", + "href": "archive/2022-08-Reykjavik-City/annotations.html#data-splitting-and-spending", + "title": "Annotations", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n\n#> [1] 0.2655087 0.3721239 0.5728534\n\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-5", - "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-5", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#what-is-wrong-with-this", + "href": "archive/2022-08-Reykjavik-City/annotations.html#what-is-wrong-with-this", + "title": "Annotations", + "section": "What is wrong with this?", + "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-6", - "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-6", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858\n\n\n\ntaxi_fit %>%\n augment(taxi_test) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.795\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#where-are-the-fitted-models", + "href": "archive/2022-08-Reykjavik-City/annotations.html#where-are-the-fitted-models", + "title": "Annotations", + "section": "Where are the fitted models?", + "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-1", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse augment() and and a metric function to compute a classification metric like brier_class().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#the-final-fit", + "href": "archive/2022-08-Reykjavik-City/annotations.html#the-final-fit", + "title": "Annotations", + "section": "The final fit", + "text": "The final fit\nSince our data spending scheme created the resamples from the training set, last_fit() will use all of the training data to fit the final workflow.\nAs shown in the Whole Game slides, there is a slightly different scheme used when we have a validation set (instead of multiple resamples like 10-fold CV)." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-7", - "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-7", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n brier_class(tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.113\n\n\n\ntaxi_fit %>%\n augment(taxi_test) %>%\n brier_class(tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.152\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#using-a-workflow", + "href": "archive/2022-08-Reykjavik-City/annotations.html#using-a-workflow", + "title": "Annotations", + "section": "Using a workflow", + "text": "Using a workflow\nWhat’s going on with the\n\nprediction from a rank-deficient fit may be misleading\n\nwarnings?\nFor linear regression, a computation is used called matrix inversion. The matrix in question is called the “model matrix” and it contains the predictor set for the training data.\nMatrix inversion can fail if two or more columns:\n\nare identical, or\nadd up to some other column.\n\nThese situations are called linear dependencies.\nWhen this happens, lm() is pretty tolerant. It does not fail but does not compute regression coefficients for a minimal number of predictors involved in the dependency (and issues the warning above).\nFor these data, there is this dependency:\nshooter_type_{level} = shooter_{level}\nHere is what is happening: since the player shooting only ever plays a single position, their indicators sum up (row-wide) to the same data as the sum of the shooter type indicators.\nIn other words, if you know the player’s name, you know their position too. This is a perfect redundency in the data.\nThe way to avoid this problem is to use step_lincomb(all_numeric_predictors()) in the recipe. This step removes the minimum number of columns to avoid the issue.\nJust in case you ever want to figure out what the specific issues are, this code might help:\n\n# Get the exact data set used to fit the model. For a recipe, that is:\nprocessed_data <- \n nhl_indicators %>% \n prep() %>% \n bake(new_data = NULL, all_predictors())\n\n# Let's capture their names:\npred_names <- names(processed_data)\n\n# caret has a function to determine the variables involved in each dependency\nissues <- caret::findLinearCombos(processed_data)\n# Convert the column index to names:\nissues_vars <- purrr::map(issues$linearCombos, ~ pred_names[.x])\n\n# caret proposes removing these columns to get rid of the issue. The choice\n# is pretty arbitrary: \nremoved_names <- pred_names[issues$remove]\n\ntl;dr\nLinear regression detects some redundancies in the predictor set. We can ignore the warnings since lm() can deal with it or use step_lincomb() to avoid the warnings." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation", - "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#per-player-statistics", + "href": "archive/2022-08-Reykjavik-City/annotations.html#per-player-statistics", + "title": "Annotations", + "section": "Per-player statistics", + "text": "Per-player statistics\nThe effect encoding method essentially takes the effect of a variable, like player, and makes a data column for that effect. In our example, the ability of a player to have an on-goal shot is quantified by a model and then added as a data column to be used in the model.\nSuppose NHL rookie Max has a single shot in the data and it was on goal. If we used a naive estimate for Max’s effect, the model is being told that Max should have a 100% chance of being on goal. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned player Davis, who has taken 250 shots and 75% of these were on goal. Davis’s proportion is more predictive because it is estimated with better data (i.e., more total shots). Partial pooling leverages the entire data set and can borrow strength from all of the players. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a player’s data is of good quality, the partial pooling effect estimate is closer to the raw proportion. Max’s data is not great and is “shrunk” towards the center of the overall on goal proportion. Since there is so little known about Max’s shot history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nIf the outcome were numeric, the effect would be the mean of the outcome per player. In this case, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-1", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#player-effects", + "href": "archive/2022-08-Reykjavik-City/annotations.html#player-effects", + "title": "Annotations", + "section": "Player effects", + "text": "Player effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for players with small sample sizes. It can’t correct for improper data usage or data leakage though." }, - { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-2", - "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-2", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" + { + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#angle", + "href": "archive/2022-08-Reykjavik-City/annotations.html#angle", + "title": "Annotations", + "section": "Angle", + "text": "Angle\nAbout geometry…\nThe coordinates for the rink are centered at (0, 0) and the goal lines are both 89 ft from center. The center of the goal on the left is at (-89, 0) and the right-hand goal is centered at (89, 0).\nFor angle to center of the goal mouth, the formula is\n\\[a = \\tan^{-1}\\left(\\frac{y}{x}\\right)\\] This is in radian units and we can convert to degrees using\n\\[a = \\frac{180}{\\pi}\\tan^{-1}\\left(\\frac{y}{x}\\right)\\] For the angle to the goal, we need to alter \\(x\\) and use x* = (89 - abs(coord_x)) instead." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-2", - "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-2", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(taxi_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6340/705]> Fold01\n#> 2 <split [6340/705]> Fold02\n#> 3 <split [6340/705]> Fold03\n#> 4 <split [6340/705]> Fold04\n#> 5 <split [6340/705]> Fold05\n#> 6 <split [6341/704]> Fold06\n#> 7 <split [6341/704]> Fold07\n#> 8 <split [6341/704]> Fold08\n#> 9 <split [6341/704]> Fold09\n#> 10 <split [6341/704]> Fold10" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#update-parameter-ranges", + "href": "archive/2022-08-Reykjavik-City/annotations.html#update-parameter-ranges", + "title": "Annotations", + "section": "Update parameter ranges", + "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nIn our case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_ns(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n\n#> # A tibble: 1 × 5\n#> name call_info source component component_id\n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_ns ns_P1Tjg\n\nns_tunable$call_info\n\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 1 15" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-3", - "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-3", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWhat is in this?\n\ntaxi_folds <- vfold_cv(taxi_train)\ntaxi_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <6340/705/7045>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <6340/705/7045>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <6340/705/7045>\n\n\nTalk about a list column, storing non-atomic types in dataframe" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#boosted-tree-tuning-parameters", + "href": "archive/2022-08-Reykjavik-City/annotations.html#boosted-tree-tuning-parameters", + "title": "Annotations", + "section": "Boosted tree tuning parameters", + "text": "Boosted tree tuning parameters\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the xgboost package.\nparsnip has its own wrapper around (xgboost::xgb.train()) called xgb_train(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the xgboost entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for xgboost)." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-4", - "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-4", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(taxi_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [5636/1409]> Fold1\n#> 2 <split [5636/1409]> Fold2\n#> 3 <split [5636/1409]> Fold3\n#> 4 <split [5636/1409]> Fold4\n#> 5 <split [5636/1409]> Fold5" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#the-final-fit-to-the-nhl-data", + "href": "archive/2022-08-Reykjavik-City/annotations.html#the-final-fit-to-the-nhl-data", + "title": "Annotations", + "section": "The final fit to the NHL data", + "text": "The final fit to the NHL data\nRecall that last_fit() uses the objects produced by initial_split() to determine what data are used for the final model fit and which are used as the test set.\nFor the validation set, last_fit() will use the non-testing data to create the final model fit. This includes the training and validation set.\nThere is no agreement in the community on whether this is the best approach or if we should just use the training set. There are good arguments either way.\nIf you only want to use the training set for the final model, you can do this via:\n\ntraining_data <- nhl_val$splits[[1]] %>% analysis()\n\n# Use `fit()` to train the model on just the training set\nfinal_glm_spline_wflow <- \n glm_spline_wflow %>% \n fit(data = training_data)\n\n# Create test set predictions\ntest_set_pred <- augment(final_glm_spline_wflow, nhl_test)\n\n# Setup and compute the test set metrics\ncls_metrics <- metric_set(roc_auc, accuracy)\n\ntest_res <- \n test_set_pred %>% \n cls_metrics(on_goal, estimate = .pred_class, .pred_yes)\ntest_res" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-5", - "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-5", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(taxi_train, strata = tip)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6340/705]> Fold01\n#> 2 <split [6340/705]> Fold02\n#> 3 <split [6340/705]> Fold03\n#> 4 <split [6340/705]> Fold04\n#> 5 <split [6340/705]> Fold05\n#> 6 <split [6340/705]> Fold06\n#> 7 <split [6341/704]> Fold07\n#> 8 <split [6341/704]> Fold08\n#> 9 <split [6341/704]> Fold09\n#> 10 <split [6342/703]> Fold10\n\n\nStratification often helps, with very little downside" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#explain-yourself", + "href": "archive/2022-08-Reykjavik-City/annotations.html#explain-yourself", + "title": "Annotations", + "section": "Explain yourself", + "text": "Explain yourself\nSome other resources:\n\nTMwR chapter Explaining Models and Predictions\nExplanatory Model Analysis book\nInterpretable Machine Learning book\nDefinitions, methods, and applications in interpretable machine learning" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-6", - "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-6", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\ntaxi_folds <- vfold_cv(taxi_train, v = 10, strata = tip)\ntaxi_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6340/705]> Fold01\n#> 2 <split [6340/705]> Fold02\n#> 3 <split [6340/705]> Fold03\n#> 4 <split [6340/705]> Fold04\n#> 5 <split [6340/705]> Fold05\n#> 6 <split [6340/705]> Fold06\n#> 7 <split [6341/704]> Fold07\n#> 8 <split [6341/704]> Fold08\n#> 9 <split [6341/704]> Fold09\n#> 10 <split [6342/703]> Fold10\n\n\nSet the seed when creating resamples" + "objectID": "archive/2022-08-Reykjavik-City/annotations.html#a-tidymodels-explainer", + "href": "archive/2022-08-Reykjavik-City/annotations.html#a-tidymodels-explainer", + "title": "Annotations", + "section": "A tidymodels explainer", + "text": "A tidymodels explainer\nFor our example, the angle was an original predictor. Recall that we made spline terms from this predictor, so there are derived features such as angle_ns_1 and so on.\nOriginal versus derived doesn’t affect local explainers since we are focused on a single prediction.\nFor global explainers, we should decide between:\n\nexplaining the overall affect of angle (lumping all its features into one importance score), or\nexplaining the effect of each term in the model (including angle_ns_1 and so on).\n\nThe choice depends on what you want. For example, if we have an original date predictor and make features for month and year, is it more informative to know if date is important (overall) or exactly how the date is important? You might want to look at it both ways." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#fit-our-model-to-the-resamples", - "href": "archive/2023-07-nyr/04-evaluating-models.html#fit-our-model-to-the-resamples", - "title": "4 - Evaluating models", - "section": "Fit our model to the resamples", - "text": "Fit our model to the resamples\n\ntaxi_res <- fit_resamples(taxi_wflow, taxi_folds)\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [6340/705]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 2 <split [6340/705]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 3 <split [6340/705]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 4 <split [6340/705]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 5 <split [6340/705]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 6 <split [6340/705]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 7 <split [6341/704]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 8 <split [6341/704]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 9 <split [6341/704]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 10 <split [6342/703]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]>" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#chicago-l-train-data", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#chicago-l-train-data", + "title": "7 - Case Study on Transportation", + "section": "Chicago L-Train data", + "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance", - "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\ntaxi_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.793 10 0.00293 Preprocessor1_Model1\n#> 2 roc_auc binary 0.809 10 0.00461 Preprocessor1_Model1\n\n\nWe can reliably measure performance using only the training data 🎉" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#predictors", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#predictors", + "title": "7 - Case Study on Transportation", + "section": "Predictors", + "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#comparing-metrics", - "href": "archive/2023-07-nyr/04-evaluating-models.html#comparing-metrics", - "title": "4 - Evaluating models", - "section": "Comparing metrics ", - "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\ntaxi_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.793 10\n#> 2 roc_auc 0.809 10\n\n\nThe ROC AUC previously was\n\n0.87 for the training set\n0.81 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#l-train-locations", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#l-train-locations", + "title": "7 - Case Study on Transportation", + "section": "L Train Locations", + "text": "L Train Locations" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-1", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_taxi <- control_resamples(save_pred = TRUE)\ntaxi_res <- fit_resamples(taxi_wflow, taxi_folds, control = ctrl_taxi)\n\ntaxi_preds <- collect_predictions(taxi_res)\ntaxi_preds\n#> # A tibble: 7,045 × 7\n#> id .pred_yes .pred_no .row .pred_class tip .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 Fold01 0.936 0.0638 10 yes no Preprocessor1_Model1\n#> 2 Fold01 0.898 0.102 20 yes no Preprocessor1_Model1\n#> 3 Fold01 0.898 0.102 47 yes no Preprocessor1_Model1\n#> 4 Fold01 0.101 0.899 51 no no Preprocessor1_Model1\n#> 5 Fold01 0.871 0.129 59 yes no Preprocessor1_Model1\n#> 6 Fold01 0.0815 0.918 60 no no Preprocessor1_Model1\n#> 7 Fold01 0.162 0.838 92 no no Preprocessor1_Model1\n#> 8 Fold01 0.26 0.74 97 no no Preprocessor1_Model1\n#> 9 Fold01 0.274 0.726 98 no no Preprocessor1_Model1\n#> 10 Fold01 0.804 0.196 104 yes no Preprocessor1_Model1\n#> # ℹ 7,035 more rows" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#your-turn-explore-the-data", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#your-turn-explore-the-data", + "title": "7 - Case Study on Transportation", + "section": "Your turn: Explore the Data", + "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n05:00" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-2", - "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-2", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\ntaxi_preds %>% \n group_by(id) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.793\n#> 2 Fold02 accuracy binary 0.8 \n#> 3 Fold03 accuracy binary 0.786\n#> 4 Fold04 accuracy binary 0.804\n#> 5 Fold05 accuracy binary 0.796\n#> 6 Fold06 accuracy binary 0.789\n#> 7 Fold07 accuracy binary 0.793\n#> 8 Fold08 accuracy binary 0.808\n#> 9 Fold09 accuracy binary 0.783\n#> 10 Fold10 accuracy binary 0.780\n#> # ℹ 20 more rows" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#splitting-with-chicago-data", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#splitting-with-chicago-data", + "title": "7 - Case Study on Transportation", + "section": "Splitting with Chicago data ", + "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#where-are-the-fitted-models", - "href": "archive/2023-07-nyr/04-evaluating-models.html#where-are-the-fitted-models", - "title": "4 - Evaluating models", - "section": "Where are the fitted models? ", - "text": "Where are the fitted models? \n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [6340/705]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [6340/705]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [6340/705]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [6340/705]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [6340/705]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [6340/705]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [6341/704]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [6341/704]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [6341/704]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [6342/703]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#time-series-resampling", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#time-series-resampling", + "title": "7 - Case Study on Transportation", + "section": "Time series resampling", + "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#bootstrapping", - "href": "archive/2023-07-nyr/04-evaluating-models.html#bootstrapping", - "title": "4 - Evaluating models", - "section": "Bootstrapping", - "text": "Bootstrapping" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#rolling-forecast-origin-resampling", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#rolling-forecast-origin-resampling", + "title": "7 - Case Study on Transportation", + "section": "Rolling forecast origin resampling", + "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#bootstrapping-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#bootstrapping-1", - "title": "4 - Evaluating models", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(taxi_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7045/2561]> Bootstrap01\n#> 2 <split [7045/2577]> Bootstrap02\n#> 3 <split [7045/2648]> Bootstrap03\n#> 4 <split [7045/2616]> Bootstrap04\n#> 5 <split [7045/2616]> Bootstrap05\n#> 6 <split [7045/2599]> Bootstrap06\n#> 7 <split [7045/2654]> Bootstrap07\n#> 8 <split [7045/2593]> Bootstrap08\n#> 9 <split [7045/2624]> Bootstrap09\n#> 10 <split [7045/2615]> Bootstrap10\n#> # ℹ 15 more rows" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling", + "title": "7 - Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game---status-update", - "href": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game---status-update", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-1", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-1", + "title": "7 - Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-3", - "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-3", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the function)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-2", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-2", + "title": "7 - Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#monte-carlo-cross-validation", - "href": "archive/2023-07-nyr/04-evaluating-models.html#monte-carlo-cross-validation", - "title": "4 - Evaluating models", - "section": "Monte Carlo Cross-Validation ", - "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(taxi_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5283/1762]> Resample01\n#> 2 <split [5283/1762]> Resample02\n#> 3 <split [5283/1762]> Resample03\n#> 4 <split [5283/1762]> Resample04\n#> 5 <split [5283/1762]> Resample05\n#> 6 <split [5283/1762]> Resample06\n#> 7 <split [5283/1762]> Resample07\n#> 8 <split [5283/1762]> Resample08\n#> 9 <split [5283/1762]> Resample09\n#> 10 <split [5283/1762]> Resample10" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-3", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-3", + "title": "7 - Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#validation-set", - "href": "archive/2023-07-nyr/04-evaluating-models.html#validation-set", - "title": "4 - Evaluating models", - "section": "Validation set ", - "text": "Validation set \n\nset.seed(853)\nvalidation_split(taxi_train, strata = tip)\n#> # Validation Set Split (0.75/0.25) using stratification \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5283/1762]> validation\n\n\nA validation set is just another type of resample" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-4", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-4", + "title": "7 - Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#random-forest-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#random-forest-1", - "title": "4 - Evaluating models", - "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this tomorrow!), as long as there are enough trees" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#our-resampling-object", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#our-resampling-object", + "title": "7 - Case Study on Transportation", + "section": "Our resampling object ", + "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#create-a-random-forest-model", - "href": "archive/2023-07-nyr/04-evaluating-models.html#create-a-random-forest-model", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#feature-engineering-with-recipes", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#feature-engineering-with-recipes", + "title": "7 - Case Study on Transportation", + "section": "Feature engineering with recipes ", + "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#create-a-random-forest-model-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#create-a-random-forest-model-1", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_wflow <- workflow(tip ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe", + "title": "7 - Case Study on Transportation", + "section": "A recipe", + "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-4", - "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-4", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates", + "title": "7 - Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-3", - "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-3", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nctrl_taxi <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, taxi_folds, control = ctrl_taxi)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.813 10 0.00305 Preprocessor1_Model1\n#> 2 roc_auc binary 0.832 10 0.00513 Preprocessor1_Model1" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates-1", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates-1", + "title": "7 - Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", - "href": "archive/2023-07-nyr/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", - "title": "4 - Evaluating models", - "section": "How can we compare multiple model workflows at once?", - "text": "How can we compare multiple model workflows at once?" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates-2", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates-2", + "title": "7 - Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set", - "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set", - "title": "4 - Evaluating models", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec))\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---remove-constant-columns", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---remove-constant-columns", + "title": "7 - Case Study on Transportation", + "section": "A recipe - remove constant columns ", + "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE) %>% \n step_zv(all_nominal_predictors())" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set-1", - "title": "4 - Evaluating models", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = taxi_folds)\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---handle-correlations", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---handle-correlations", + "title": "7 - Case Study on Transportation", + "section": "A recipe - handle correlations ", + "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set-2", - "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set-2", - "title": "4 - Evaluating models", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = taxi_folds) %>%\n rank_results()\n#> # A tibble: 4 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_for… Prepro… accura… 0.813 0.00339 10 formula rand… 1\n#> 2 formula_rand_for… Prepro… roc_auc 0.833 0.00528 10 formula rand… 1\n#> 3 formula_decision… Prepro… accura… 0.793 0.00293 10 formula deci… 2\n#> 4 formula_decision… Prepro… roc_auc 0.809 0.00461 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#make-some-models", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#make-some-models", + "title": "7 - Case Study on Transportation", + "section": "Make some models ", + "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) %>% set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) %>% \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-5", - "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-5", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nWhen do you think a workflow set would be useful?\n\n\n\n−+\n03:00" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#process-them-on-the-resamples", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#process-them-on-the-resamples", + "title": "7 - Case Study on Transportation", + "section": "Process them on the resamples", + "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set %>% \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game---status-update-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game---status-update-1", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#how-do-the-results-look", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#how-do-the-results-look", + "title": "7 - Case Study on Transportation", + "section": "How do the results look?", + "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 31 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.798 0.104 16 recipe cubis… 1\n#> 2 pca_cubist Preprocessor3_Model3 mae 0.978 0.110 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor4_Model2 mae 0.983 0.122 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model1 mae 0.991 0.127 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor3_Model2 mae 0.991 0.113 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor2_Model2 mae 1.02 0.118 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor1_Model3 mae 1.05 0.134 16 recipe cubis… 7\n#> 8 basic_cubist Preprocessor1_Model8 mae 1.07 0.115 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model7 mae 1.07 0.112 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model6 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 21 more rows" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#the-final-fit", - "href": "archive/2023-07-nyr/04-evaluating-models.html#the-final-fit", - "title": "4 - Evaluating models", - "section": "The final fit ", - "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# taxi_split has train + test info\nfinal_fit <- last_fit(rf_wflow, taxi_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [7045/1762]> train/test split <tibble> <tibble> <tibble> <workflow>" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#plot-the-results", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#plot-the-results", + "title": "7 - Case Study on Transportation", + "section": "Plot the results ", + "text": "Plot the results \n\nautoplot(chi_res)" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit", - "href": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.810 Preprocessor1_Model1\n#> 2 roc_auc binary 0.817 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#pull-out-specific-results", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#pull-out-specific-results", + "title": "7 - Case Study on Transportation", + "section": "Pull out specific results ", + "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n autoplot()" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit-1", - "href": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit-1", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 1,762 × 7\n#> id .pred_yes .pred_no .row .pred_class tip .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 train/test split 0.732 0.268 10 yes no Preprocessor1_Mo…\n#> 2 train/test split 0.827 0.173 29 yes yes Preprocessor1_Mo…\n#> 3 train/test split 0.899 0.101 35 yes yes Preprocessor1_Mo…\n#> 4 train/test split 0.914 0.0856 42 yes yes Preprocessor1_Mo…\n#> 5 train/test split 0.911 0.0889 47 yes no Preprocessor1_Mo…\n#> 6 train/test split 0.848 0.152 54 yes yes Preprocessor1_Mo…\n#> 7 train/test split 0.580 0.420 59 yes yes Preprocessor1_Mo…\n#> 8 train/test split 0.912 0.0876 62 yes yes Preprocessor1_Mo…\n#> 9 train/test split 0.810 0.190 63 yes yes Preprocessor1_Mo…\n#> 10 train/test split 0.960 0.0402 69 yes yes Preprocessor1_Mo…\n#> # ℹ 1,752 more rows" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit", + "title": "7 - Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit-2", - "href": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit-2", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 7045 \n#> Number of independent variables: 6 \n#> Mtry: 2 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.1373147\n\n\nUse this for prediction on new data, like for deploying" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-1", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-1", + "title": "7 - Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game", - "href": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game", - "title": "4 - Evaluating models", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-2", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-2", + "title": "7 - Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-6", - "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-6", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nEnd of the day discussion!\nWhich model do you think you would decide to use?\nWhat surprised you the most?\nWhat is one thing you are looking forward to for tomorrow?\n\n\n\n−+\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-3", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-3", + "title": "7 - Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips", - "href": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips", - "title": "2 - Your data budget", - "section": "Data on Chicago taxi trips", - "text": "Data on Chicago taxi trips\n\n\n\nThe city of Chicago releases anonymized trip-level data on taxi trips in the city.\nWe pulled a sample of 10,000 rides occurring in early 2022.\nType ?modeldatatoo::data_taxi() to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/8322/taxi" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-4", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-4", + "title": "7 - Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#which-of-these-variables-can-we-use", - "href": "archive/2023-07-nyr/02-data-budget.html#which-of-these-variables-can-we-use", - "title": "2 - Your data budget", - "section": "Which of these variables can we use?", - "text": "Which of these variables can we use?\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\n\ntaxi <- data_taxi()\n\nnames(taxi)\n#> [1] \"tip\" \"id\" \"duration\" \"distance\" \"fare\" \n#> [6] \"tolls\" \"extras\" \"total_cost\" \"payment_type\" \"company\" \n#> [11] \"local\" \"dow\" \"month\" \"hour\"" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-5", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-5", + "title": "7 - Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#checklist-for-predictors", - "href": "archive/2023-07-nyr/02-data-budget.html#checklist-for-predictors", - "title": "2 - Your data budget", - "section": "Checklist for predictors", - "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#building-a-model-stack", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#building-a-model-stack", + "title": "7 - Case Study on Transportation", + "section": "Building a model stack ", + "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-1", - "href": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-1", - "title": "2 - Your data budget", - "section": "Data on Chicago taxi trips", - "text": "Data on Chicago taxi trips\nWe are using a slightly modified version from the modeldatatoo data.\n\ntaxi <- taxi %>%\n mutate(month = factor(month, levels = c(\"Jan\", \"Feb\", \"Mar\", \"Apr\"))) %>% \n select(-c(id, duration, fare, tolls, extras, total_cost, payment_type)) %>% \n drop_na()" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#start-the-stack-and-add-members", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#start-the-stack-and-add-members", + "title": "7 - Case Study on Transportation", + "section": "Start the stack and add members ", + "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() %>% \n add_candidates(chi_res)" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-2", - "href": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-2", - "title": "2 - Your data budget", - "section": "Data on Chicago taxi trips", - "text": "Data on Chicago taxi trips\n\n\n\nN = 10,000\nA nominal outcome, tip, with levels \"yes\" and \"no\"\n6 other variables\n\ncompany, local, and dow, and month are nominal predictors\ndistance and hours are numeric predictors\n\n\n\n\n\n\nCredit: https://unsplash.com/photos/7_r85l4eht8\n\n\ntip: Whether the rider left a tip. A factor with levels “yes” and “no”.\ndistance: The trip distance, in odometer miles.\ncompany: The taxi company, as a factor. Companies that occurred few times were binned as “other”.\nlocal: Whether the trip started in the same community area as it began. See the source data for community area values.\ndow: The day of the week in which the trip began, as a factor.\nmonth: The month in which the trip began, as a factor.\nhour: The hour of the day in which the trip began, as a numeric." + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#estimate-weights-for-each-candidate", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#estimate-weights-for-each-candidate", + "title": "7 - Case Study on Transportation", + "section": "Estimate weights for each candidate ", + "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack)\n\nchi_stack_res\n#> # A tibble: 5 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.343\n#> 2 pca_cubist_3_2 cubist_rules 0.236\n#> 3 basic_cubist_1_4 cubist_rules 0.189\n#> 4 pca_lm_4_1 linear_reg 0.163\n#> 5 pca_cubist_3_3 cubist_rules 0.109" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-3", - "href": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-3", - "title": "2 - Your data budget", - "section": "Data on Chicago taxi trips", - "text": "Data on Chicago taxi trips\n\ntaxi\n#> # A tibble: 8,807 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 1.24 Sun Taxi no Thu Feb 13\n#> 2 no 5.39 Flash Cab no Sat Mar 12\n#> 3 yes 3.01 City Service no Wed Feb 17\n#> 4 no 18.4 Sun Taxi no Sat Apr 6\n#> 5 yes 1.76 Sun Taxi no Sun Jan 15\n#> 6 yes 13.6 Sun Taxi no Mon Feb 17\n#> 7 yes 3.71 City Service no Mon Mar 21\n#> 8 yes 4.8 other no Tue Mar 9\n#> 9 yes 18.0 City Service no Fri Jan 19\n#> 10 no 17.5 other yes Thu Apr 12\n#> # ℹ 8,797 more rows" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#how-did-it-do", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#how-did-it-do", + "title": "7 - Case Study on Transportation", + "section": "How did it do? ", + "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending", - "href": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#what-does-it-use", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#what-does-it-use", + "title": "7 - Case Study on Transportation", + "section": "What does it use? ", + "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-1", - "href": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-1", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#fit-the-required-candidate-models", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#fit-the-required-candidate-models", + "title": "7 - Case Study on Transportation", + "section": "Fit the required candidate models", + "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-2", - "href": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-2", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#the-test-set-best-cubist-model", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#the-test-set-best-cubist-model", + "title": "7 - Case Study on Transportation", + "section": "The test set: best Cubist model ", + "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n select_best()\n\ncubist_res <- \n chi_res %>% \n extract_workflow(\"pca_cubist\") %>% \n finalize_workflow(best_cubist) %>% \n last_fit(split = chi_split, metrics = metric_set(mae))" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#your-turn", - "href": "archive/2023-07-nyr/02-data-budget.html#your-turn", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#the-test-set-stack-ensemble", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#the-test-set-stack-ensemble", + "title": "7 - Case Study on Transportation", + "section": "The test set: stack ensemble", + "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) %>% \n bind_cols(chi_test)" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#the-initial-split", - "href": "archive/2023-07-nyr/02-data-budget.html#the-initial-split", - "title": "2 - Your data budget", - "section": "The initial split ", - "text": "The initial split \n\nset.seed(123)\ntaxi_split <- initial_split(taxi)\ntaxi_split\n#> <Training/Testing/Total>\n#> <6605/2202/8807>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#compare-the-results", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#compare-the-results", + "title": "7 - Case Study on Transportation", + "section": "Compare the results ", + "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.670 Preprocessor1_Model1\n\nstack_pred %>% mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.689" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#accessing-the-data", - "href": "archive/2023-07-nyr/02-data-budget.html#accessing-the-data", - "title": "2 - Your data budget", - "section": "Accessing the data ", - "text": "Accessing the data \n\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)" + "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#plot-the-test-set", + "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#plot-the-test-set", + "title": "7 - Case Study on Transportation", + "section": "Plot the test set ", + "text": "Plot the test set \n\n\ncubist_res %>% \n collect_predictions() %>% \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#the-training-set", - "href": "archive/2023-07-nyr/02-data-budget.html#the-training-set", - "title": "2 - Your data budget", - "section": "The training set", - "text": "The training set\n\ntaxi_train\n#> # A tibble: 6,605 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 4.54 City Service no Sat Mar 16\n#> 2 no 10.2 Flash Cab no Mon Feb 8\n#> 3 yes 12.4 other no Sun Apr 15\n#> 4 yes 15.3 Sun Taxi no Mon Apr 18\n#> 5 no 6.41 Flash Cab no Wed Apr 14\n#> 6 yes 1.56 other no Tue Jan 13\n#> 7 yes 3.13 Flash Cab no Sun Apr 12\n#> 8 yes 7.54 other no Tue Apr 8\n#> 9 yes 6.98 Flash Cab no Tue Apr 5\n#> 10 yes 0.7 Taxi Affiliation Services no Tue Jan 9\n#> # ℹ 6,595 more rows" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#working-with-our-predictors", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#working-with-our-predictors", + "title": "5 - Feature engineering", + "section": "Working with our predictors", + "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for lm()).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#the-test-set", - "href": "archive/2023-07-nyr/02-data-budget.html#the-test-set", - "title": "2 - Your data budget", - "section": "The test set ", - "text": "The test set \n🙈\n\nThere are 2202 rows and 7 columns in the test set." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-is-feature-engineering", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-is-feature-engineering", + "title": "5 - Feature engineering", + "section": "What is feature engineering?", + "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPCA feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection." }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#your-turn-1", - "href": "archive/2023-07-nyr/02-data-budget.html#your-turn-1", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n−+\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#example-dates", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#example-dates", + "title": "5 - Feature engineering", + "section": "Example: Dates", + "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen a date column is used in its native format, it is usually converted by an R model to an integer.\n\n\nIt can be re-engineered as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-3", - "href": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-3", - "title": "2 - Your data budget", - "section": "Data splitting and spending ", - "text": "Data splitting and spending \n\nset.seed(123)\ntaxi_split <- initial_split(taxi, prop = 0.8)\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)\n\nnrow(taxi_train)\n#> [1] 7045\nnrow(taxi_test)\n#> [1] 1762" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#general-definitions", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#general-definitions", + "title": "5 - Feature engineering", + "section": "General definitions ", + "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\nIn a little bit, we’ll see successful (and unsuccessful) feature engineering methods for our example data.\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#section-1", - "href": "archive/2023-07-nyr/02-data-budget.html#section-1", - "title": "2 - Your data budget", - "section": "", - "text": "We will use this tomorrow" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#the-nhl-data", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#the-nhl-data", + "title": "5 - Feature engineering", + "section": "The NHL data 🏒", + "text": "The NHL data 🏒\n\nFrom Pittsburgh Penguins games, 8,915 shots\nData from the 2015-2016 season\n\n\nLet’s predict whether a shot is on-goal (a goal or blocked by goaltender) or not." }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#your-turn-2", - "href": "archive/2023-07-nyr/02-data-budget.html#your-turn-2", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nExplore the taxi_train data on your own!\n\nWhat’s the distribution of the outcome, tip?\nWhat’s the distribution of numeric variables like distance?\nHow does tip differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#case-study", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#case-study", + "title": "5 - Feature engineering", + "section": "Case study", + "text": "Case study\n\nlibrary(tidymodels)\nlibrary(ongoal)\n\ntidymodels_prefer()\n\nglimpse(season_2015)\n#> Rows: 8,915\n#> Columns: 13\n#> $ on_goal <fct> no, no, yes, no, yes, no, no, yes, no, no, no, yes, no…\n#> $ period <int> 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, …\n#> $ game_seconds <dbl> 13, 36, 47, 92, 99, 125, 179, 220, 252, 309, 413, 427,…\n#> $ strength <fct> even, even, even, even, even, even, even, even, even, …\n#> $ home_skaters <dbl> 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, …\n#> $ away_skaters <dbl> 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, …\n#> $ goaltender <fct> marc_andre_fleury, antti_niemi, antti_niemi, antti_nie…\n#> $ goal_difference <dbl> 0, 0, 0, 0, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, -1, -1, 1,…\n#> $ shooter <fct> evgeni_malkin, valeri_nichushkin, phil_kessel, beau_be…\n#> $ shooter_type <fct> center, right_wing, center, right_wing, center, defens…\n#> $ coord_x <dbl> -66, -49, 64, 65, 80, 42, -55, 62, -67, -58, 76, 56, -…\n#> $ coord_y <dbl> -11, -21, -31, -21, 13, 31, -19, 15, -9, -16, -8, 25, …\n#> $ extra_attacker <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#section-3", - "href": "archive/2023-07-nyr/02-data-budget.html#section-3", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n ggplot(aes(x = tip)) +\n geom_bar()" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#data-splitting-strategy", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#data-splitting-strategy", + "title": "5 - Feature engineering", + "section": "Data splitting strategy", + "text": "Data splitting strategy" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#section-4", - "href": "archive/2023-07-nyr/02-data-budget.html#section-4", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n ggplot(aes(x = tip, fill = local)) +\n geom_bar() +\n scale_fill_viridis_d(end = .5)" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#why-a-validation-set", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#why-a-validation-set", + "title": "5 - Feature engineering", + "section": "Why a validation set?", + "text": "Why a validation set?\nRecall that resampling gives us performance measures without using the test set.\nIt’s important to get good resampling statistics (e.g. \\(R^2\\)).\n\nThat usually means having enough data to estimate performance.\n\nWhen you have “a lot” of data, a validation set can be an efficient way to do this." }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#section-5", - "href": "archive/2023-07-nyr/02-data-budget.html#section-5", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n mutate(tip = forcats::fct_rev(tip)) %>% \n ggplot(aes(x = hour, fill = tip)) +\n geom_bar()" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#splitting-the-nhl-data", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#splitting-the-nhl-data", + "title": "5 - Feature engineering", + "section": "Splitting the NHL data ", + "text": "Splitting the NHL data \n\nset.seed(23)\nnhl_split <- initial_split(season_2015, prop = 3/4)\nnhl_split\n#> <Training/Testing/Total>\n#> <6686/2229/8915>\n\nnhl_train_and_val <- training(nhl_split)\nnhl_test <- testing(nhl_split)\n\n## not testing\nnrow(nhl_train_and_val)\n#> [1] 6686\n \n## testing\nnrow(nhl_test)\n#> [1] 2229" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#section-6", - "href": "archive/2023-07-nyr/02-data-budget.html#section-6", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n mutate(tip = forcats::fct_rev(tip)) %>% \n ggplot(aes(x = hour, fill = tip)) +\n geom_bar(position = \"fill\")" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#validation-split", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#validation-split", + "title": "5 - Feature engineering", + "section": "Validation split ", + "text": "Validation split \nSince there are a lot of observations, we’ll use a validation set:\n\nset.seed(234)\nnhl_val <- validation_split(nhl_train_and_val, prop = 0.80)\nnhl_val\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5348/1338]> validation\n\n\nRemember that a validation split is a type of resample." }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#section-7", - "href": "archive/2023-07-nyr/02-data-budget.html#section-7", - "title": "2 - Your data budget", - "section": "", - "text": "taxi_train %>% \n mutate(tip = forcats::fct_rev(tip)) %>% \n ggplot(aes(x = distance)) +\n geom_histogram(bins = 100) +\n facet_grid(vars(tip))" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn", + "title": "5 - Feature engineering", + "section": "Your turn", + "text": "Your turn\n\nLet’s explore the training set data.\nUse the function plot_nhl_shots() for nice spatial plots of the data.\n\n\n\nnhl_train <- analysis(nhl_val$splits[[1]])\n\nset.seed(100)\nnhl_train %>% \n sample_n(200) %>%\n plot_nhl_shots(emphasis = shooter_type)\n\n\n\n\n\n\n\n\n\n\n\n\n08:00" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#section-8", - "href": "archive/2023-07-nyr/02-data-budget.html#section-8", - "title": "2 - Your data budget", - "section": "", - "text": "Stratified sampling would split within response values\n\nBased on our EDA, we know that the source data contains fewer \"no\" tip values than \"yes\". We want to make sure we allot equal proportions of those responses so that both the training and testing data have enough of each to give accurate estimates." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#prepare-your-data-for-modeling", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#prepare-your-data-for-modeling", + "title": "5 - Feature engineering", + "section": "Prepare your data for modeling ", + "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of feature engineering steps that provide preprocessing tools to be applied to data.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#stratification", - "href": "archive/2023-07-nyr/02-data-budget.html#stratification", - "title": "2 - Your data budget", - "section": "Stratification", - "text": "Stratification\nUse strata = tip\n\nset.seed(123)\ntaxi_split <- initial_split(taxi, prop = 0.8, strata = tip)\ntaxi_split\n#> <Training/Testing/Total>\n#> <7045/1762/8807>" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#a-first-recipe", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#a-first-recipe", + "title": "5 - Feature engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#stratification-1", - "href": "archive/2023-07-nyr/02-data-budget.html#stratification-1", - "title": "2 - Your data budget", - "section": "Stratification", - "text": "Stratification\nStratification often helps, with very little downside" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#a-first-recipe-1", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#a-first-recipe-1", + "title": "5 - Feature engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nsummary(nhl_rec)\n#> # A tibble: 13 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 period <chr [2]> predictor original\n#> 2 game_seconds <chr [2]> predictor original\n#> 3 strength <chr [3]> predictor original\n#> 4 home_skaters <chr [2]> predictor original\n#> 5 away_skaters <chr [2]> predictor original\n#> 6 goaltender <chr [3]> predictor original\n#> 7 goal_difference <chr [2]> predictor original\n#> 8 shooter <chr [3]> predictor original\n#> 9 shooter_type <chr [3]> predictor original\n#> 10 coord_x <chr [2]> predictor original\n#> 11 coord_y <chr [2]> predictor original\n#> 12 extra_attacker <chr [2]> predictor original\n#> 13 on_goal <chr [3]> outcome original" }, { - "objectID": "archive/2023-07-nyr/02-data-budget.html#the-whole-game---status-update", - "href": "archive/2023-07-nyr/02-data-budget.html#the-whole-game---status-update", - "title": "2 - Your data budget", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#create-indicator-variables", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#create-indicator-variables", + "title": "5 - Feature engineering", + "section": "Create indicator variables ", + "text": "Create indicator variables \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns." }, { - "objectID": "archive/2022-08-Reykjavik-City/index.html", - "href": "archive/2022-08-Reykjavik-City/index.html", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels presented in Reykjavík Iceland. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#filter-out-constant-columns", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#filter-out-constant-columns", + "title": "5 - Feature engineering", + "section": "Filter out constant columns ", + "text": "Filter out constant columns \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" }, { - "objectID": "archive/2022-08-Reykjavik-City/index.html#welcome", - "href": "archive/2022-08-Reykjavik-City/index.html#welcome", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels presented in Reykjavík Iceland. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#normalization", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#normalization", + "title": "5 - Feature engineering", + "section": "Normalization ", + "text": "Normalization \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." }, { - "objectID": "archive/2022-08-Reykjavik-City/index.html#is-this-workshop-for-me", - "href": "archive/2022-08-Reykjavik-City/index.html#is-this-workshop-for-me", - "title": "Machine learning with tidymodels", - "section": "Is this workshop for me? ", - "text": "Is this workshop for me? \nThis course assumes intermediate R knowledge. This workshop is for you if:\n\nYou can use the magrittr pipe %>% and/or native pipe |>\nYou are familiar with functions from dplyr, tidyr, and ggplot2\nYou can read data into R, transform and reshape data, and make a wide variety of graphs\n\nWe expect participants to have some exposure to basic statistical concepts, but NOT intermediate or expert familiarity with modeling or machine learning." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#reduce-correlation", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#reduce-correlation", + "title": "5 - Feature engineering", + "section": "Reduce correlation ", + "text": "Reduce correlation \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." }, { - "objectID": "archive/2022-08-Reykjavik-City/index.html#preparation", - "href": "archive/2022-08-Reykjavik-City/index.html#preparation", - "title": "Machine learning with tidymodels", - "section": "Preparation", - "text": "Preparation\nPlease join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2022.02), available at https://www.rstudio.com/download\nThe following R packages, which you can install from the R console:\n\n\ninstall.packages(c(\"Cubist\", \"DALEXtra\", \"doParallel\", \"earth\", \"embed\", \n \"forcats\", \"lme4\", \"parallelly\", \"ranger\", \"remotes\", \"rpart\", \n \"rpart.plot\", \"rules\", \"stacks\", \"tidymodels\",\n \"vetiver\", \"xgboost\"))\n\nremotes::install_github(\"topepo/ongoal@hockeyR\")" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps", + "title": "5 - Feature engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" }, { - "objectID": "archive/2022-08-Reykjavik-City/index.html#slides", - "href": "archive/2022-08-Reykjavik-City/index.html#slides", - "title": "Machine learning with tidymodels", - "section": "Slides", - "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. There are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nDay One\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n\n\n\nDay Two\n\n05: Feature engineering\n06: Tuning hyperparameters\n07: Transit Case Study\n08: Wrapping up\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps-1", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps-1", + "title": "5 - Feature engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = on_goal)\n\n\nA fancy machine learning supervised dimension reduction technique…\n\nNote that this uses the outcome, and it is from an extension package" }, { - "objectID": "archive/2022-08-Reykjavik-City/index.html#code", - "href": "archive/2022-08-Reykjavik-City/index.html#code", - "title": "Machine learning with tidymodels", - "section": "Code", - "text": "Code\nQuarto files (version 1.4.104) for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps-2", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps-2", + "title": "5 - Feature engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_ns(coord_y, coord_x, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" }, { - "objectID": "archive/2022-08-Reykjavik-City/index.html#past-workshops", - "href": "archive/2022-08-Reykjavik-City/index.html#past-workshops", - "title": "Machine learning with tidymodels", - "section": "Past workshops", - "text": "Past workshops\n\n25-26 July 2022 at rstudio::conf()\n16-17 August 2022 in Reykjavik Iceland" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-1", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-1", + "title": "5 - Feature engineering", + "section": "Your turn", + "text": "Your turn\n\nCreate a recipe() for the on-goal data to :\n\ncreate one-hot indicator variables\nremove zero-variance variables\n\n\n\n\n03:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/index.html#acknowledgments", - "href": "archive/2022-08-Reykjavik-City/index.html#acknowledgments", - "title": "Machine learning with tidymodels", - "section": "Acknowledgments", - "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#minimal-recipe", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#minimal-recipe", + "title": "5 - Feature engineering", + "section": "Minimal recipe ", + "text": "Minimal recipe \n\nnhl_indicators <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())" }, { - "objectID": "archive/2022-08-Reykjavik-City/index.html#reuse-and-licensing", - "href": "archive/2022-08-Reykjavik-City/index.html#reuse-and-licensing", - "title": "Machine learning with tidymodels", - "section": "Reuse and licensing", - "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#using-a-workflow", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#using-a-workflow", + "title": "5 - Feature engineering", + "section": "Using a workflow ", + "text": "Using a workflow \n\nset.seed(9)\n\nnhl_glm_wflow <-\n workflow() %>%\n add_recipe(nhl_indicators) %>%\n add_model(logistic_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nnhl_glm_res <-\n nhl_glm_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_glm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.756 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.753 1 NA Preprocessor1_Model1" }, { - "objectID": "archive/2022-08-Reykjavik-City/08-wrapping-up.html#your-turn", - "href": "archive/2022-08-Reykjavik-City/08-wrapping-up.html#your-turn", - "title": "8 - Wrapping up", + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-2", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-2", + "title": "5 - Feature engineering", "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n05:00" - }, - { - "objectID": "archive/2022-08-Reykjavik-City/08-wrapping-up.html#resources-to-keep-learning", - "href": "archive/2022-08-Reykjavik-City/08-wrapping-up.html#resources-to-keep-learning", - "title": "8 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Twitter and at the tidyverse blog for updates!" + "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-parameters", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-parameters", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#holdout-predictions", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#holdout-predictions", + "title": "5 - Feature engineering", + "section": "Holdout predictions ", + "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nglm_val_pred <- collect_predictions(nhl_glm_res)\nglm_val_pred %>% slice(1:7)\n#> # A tibble: 7 × 7\n#> id .pred_yes .pred_no .row .pred_class on_goal .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 validation 0.00844 0.992 2 no no Preprocessor1_Mod…\n#> 2 validation 0.0864 0.914 10 no no Preprocessor1_Mod…\n#> 3 validation 0.799 0.201 22 yes yes Preprocessor1_Mod…\n#> 4 validation 0.615 0.385 24 yes yes Preprocessor1_Mod…\n#> 5 validation 0.787 0.213 31 yes yes Preprocessor1_Mod…\n#> 6 validation 0.872 0.128 39 yes yes Preprocessor1_Mod…\n#> 7 validation 0.00000000903 1.00 40 no no Preprocessor1_Mod…" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#optimize-tuning-parameters", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#optimize-tuning-parameters", - "title": "6 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#two-class-data-1", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#two-class-data-1", + "title": "5 - Feature engineering", + "section": "Two class data", + "text": "Two class data\nThese definitions assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "title": "6 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#varying-the-threshold", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#varying-the-threshold", + "title": "5 - Feature engineering", + "section": "Varying the threshold", + "text": "Varying the threshold" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choosing-tuning-parameters", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choosing-tuning-parameters", - "title": "6 - Tuning Hyperparameters", - "section": "Choosing tuning parameters ", - "text": "Choosing tuning parameters \nLet’s take our previous recipe and add a few changes:\n\nglm_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_lencode_mixed(shooter, goaltender, outcome = vars(on_goal)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_mutate(\n angle = abs( atan2(abs(coord_y), (89 - coord_x) ) * (180 / pi) ),\n defensive_zone = ifelse(coord_x <= -25.5, 1, 0),\n behind_goal_line = ifelse(coord_x >= 89, 1, 0)\n ) %>%\n step_zv(all_predictors()) %>%\n step_ns(angle, deg_free = tune(\"angle\")) %>%\n step_ns(coord_x, deg_free = tune(\"coord_x\")) %>%\n step_normalize(all_numeric_predictors())\n\n\nLet’s tune() our spline terms!" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curves", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curves", + "title": "5 - Feature engineering", + "section": "ROC curves", + "text": "ROC curves\nTo make an ROC (receiver operator characteristic) curve, we:\n\ncalculate the sensitivity and specificity for all possible thresholds\nplot false positive rate (x-axis) versus true positive rate (y-axis)\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choosing-tuning-parameters-1", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choosing-tuning-parameters-1", - "title": "6 - Tuning Hyperparameters", - "section": "Choosing tuning parameters ", - "text": "Choosing tuning parameters \nLet’s take our previous recipe and add a few changes:\n\nglm_spline_wflow <-\n workflow() %>%\n add_model(logistic_reg()) %>%\n add_recipe(glm_rec)" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curves-1", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curves-1", + "title": "5 - Feature engineering", + "section": "ROC curves ", + "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\nroc_curve_points <- glm_val_pred %>% roc_curve(truth = on_goal, .pred_yes)\nroc_curve_points %>% slice(1, 50, 100)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.00143 0.0412 0.968\n#> 3 0.00988 0.127 0.968\n\nglm_val_pred %>% roc_auc(truth = on_goal, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.753" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#section", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#section", - "title": "6 - Tuning Hyperparameters", - "section": "", - "text": "Splines replace the existing numeric predictor with a set of columns that allow a model to emulate a flexible, nonlinear relationship.\nMore spline terms = more “wiggly”, i.e. flexibly model a nonlinear relationship\nHow many spline terms? This is called degrees of freedom\n2 and 5 look like they underfit; 20 and 100 look like they overfit" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curve-plot", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curve-plot", + "title": "5 - Feature engineering", + "section": "ROC curve plot ", + "text": "ROC curve plot \n\nautoplot(roc_curve_points)" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#splines-and-nonlinear-relationships", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#splines-and-nonlinear-relationships", - "title": "6 - Tuning Hyperparameters", - "section": "Splines and nonlinear relationships", - "text": "Splines and nonlinear relationships\n\n\nOur hockey data exhibits nonlinear relationships\nWe can model nonlinearity like this via a model (later this afternoon) or feature engineering\nHow do we decide how “wiggly” or flexible to make our spline features? TUNING" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-3", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-3", + "title": "5 - Feature engineering", + "section": "Your turn", + "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#grid-search", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#grid-search", - "title": "6 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nParameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-do-we-do-with-the-player-data", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-do-we-do-with-the-player-data", + "title": "5 - Feature engineering", + "section": "What do we do with the player data? 🏒", + "text": "What do we do with the player data? 🏒\nThere are 598 unique player values in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump players who rarely shoot into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the shooter column with the estimated effect of that predictor\n\n\n\nLet’s look at othering then effect encodings." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid", - "title": "6 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\nglm_spline_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 2 parameters for tuning\n#> \n#> identifier type object\n#> angle deg_free nparam[+]\n#> coord_x deg_free nparam[+]\n\n\nA parameter set can be updated (e.g. to change the ranges)." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#per-player-statistics", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#per-player-statistics", + "title": "5 - Feature engineering", + "section": "Per-player statistics", + "text": "Per-player statistics" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid-1", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid-1", - "title": "6 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\n\n\nset.seed(12)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 2\n#> angle coord_x\n#> <int> <int>\n#> 1 5 8\n#> 2 1 13\n#> 3 8 6\n#> 4 11 5\n#> 5 4 1\n#> 6 6 10\n#> 7 10 15\n#> 8 12 13\n#> 9 8 13\n#> 10 10 6\n#> # ℹ 15 more rows\n\n\n\n\nA space-filling design like this tends to perform better than random grids.\nSpace-filling designs are also usually more efficient than regular grids." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#collapsing-factor-levels", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#collapsing-factor-levels", + "title": "5 - Feature engineering", + "section": "Collapsing factor levels", + "text": "Collapsing factor levels\nThere is a recipe step that will redefine factor levels based on the their frequency in the training set:\n\nnhl_other_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n # Any player with <= 0.01% of shots is set to \"other\"\n step_other(shooter, threshold = 0.001) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\nUsing this code, 402 players (out of 598) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n03:00" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#does-othering-help", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#does-othering-help", + "title": "5 - Feature engineering", + "section": "Does othering help?", + "text": "Does othering help?\n\nnhl_other_wflow <-\n nhl_glm_wflow %>%\n update_recipe(nhl_other_rec)\n\nnhl_other_res <-\n nhl_other_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.778 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.804 1 NA Preprocessor1_Model1\n\nA little better ROC AUC and much faster to complete.\nNow let’s look at a more sophisticated tool called effect encodings." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid-2", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid-2", - "title": "6 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\nset.seed(12)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 25)\n\ngrid\n#> # A tibble: 225 × 2\n#> angle coord_x\n#> <int> <int>\n#> 1 1 1\n#> 2 2 1\n#> 3 3 1\n#> 4 4 1\n#> 5 5 1\n#> 6 6 1\n#> 7 7 1\n#> 8 8 1\n#> 9 9 1\n#> 10 10 1\n#> # ℹ 215 more rows\n\n\nNote that even though we requested 25x25=625 rows, we only got 15x15=225 back, since the deg_free parameters only have a range of 1->15." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-is-an-effect-encoding", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-is-an-effect-encoding", + "title": "5 - Feature engineering", + "section": "What is an effect encoding?", + "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> on_goal shooter .row\n#> <fct> <fct> <int>\n#> 1 yes sidney_crosby 1\n#> 2 yes mike_hoffman 2\n#> 3 yes ian_cole 3\n#> 4 yes matt_cullen 4\n#> 5 yes kris_letang 5\n#> 6 no nazem_kadri 6\n#> 7 yes david_perron 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> on_goal shooter .row\n#> <fct> <dbl> <int>\n#> 1 yes 0.220 1\n#> 2 yes 0.302 2\n#> 3 yes 0.261 3\n#> 4 yes 0.463 4\n#> 5 yes 0.197 5\n#> 6 no 0.302 6\n#> 7 yes 0.400 7\n\n\nThe shooter column is replaced with the log-odds of being on goal.\n\nAs a reminder:\n\\[\\text{log-odds} = log\\left(\\frac{\\hat{p}}{1 - \\hat{p}}\\right)\\]\nwhere \\(\\hat{p}\\) is the on goal rate estimate.\nFor logistic regression, this is what the predictors are modeling. The log-odds are more likely to be linear with the outcome." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#update-parameter-ranges", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#update-parameter-ranges", - "title": "6 - Tuning Hyperparameters", - "section": "Update parameter ranges ", - "text": "Update parameter ranges \n\nset.seed(12)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n update(angle = spline_degree(c(2L, 50L)),\n coord_x = spline_degree(c(2L, 50L))) %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 2\n#> angle coord_x\n#> <int> <int>\n#> 1 14 27\n#> 2 4 42\n#> 3 26 20\n#> 4 36 16\n#> 5 13 3\n#> 6 20 33\n#> 7 31 49\n#> 8 40 44\n#> 9 24 45\n#> 10 34 18\n#> # ℹ 15 more rows\n\n\nEven though angle is a deg_free parameter in step_ns(), we don’t use the dials deg_free() object here. We have a special spline_degree() function that has better defaults for splines." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#per-player-statistics-again", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#per-player-statistics-again", + "title": "5 - Feature engineering", + "section": "Per-player statistics again", + "text": "Per-player statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these rates use partial pooling.\nPooling borrows strength across players and shrinks extreme values (e.g. zero or one) towards the mean for players with very few shots.\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for players with fewer shots by shrinking the estimate to the overall on-goal rate (56.5%)" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#the-results", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#the-results", - "title": "6 - Tuning Hyperparameters", - "section": "The results ", - "text": "The results \n\n\ngrid %>% \n ggplot(aes(angle, coord_x)) +\n geom_point(size = 4)" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#partial-pooling", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#partial-pooling", + "title": "5 - Feature engineering", + "section": "Partial pooling", + "text": "Partial pooling" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#spline-grid-search", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#spline-grid-search", - "title": "6 - Tuning Hyperparameters", - "section": "Spline grid search ", - "text": "Spline grid search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE, parallel_over = \"everything\")\n\nglm_spline_res <-\n glm_spline_wflow %>%\n tune_grid(resamples = nhl_val, grid = grid, control = ctrl)\n\nglm_spline_res\n#> # Tuning results\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [5348/1338]> validation <tibble [50 × 6]> <tibble [2 × 3]> <tibble [33,450 × 8]>\n#> \n#> There were issues with some computations:\n#> \n#> - Warning(s) x1: glm.fit: algorithm did not converge, glm.fit: fitted probabilities numerically 0 or 1 occurred\n#> - Warning(s) x1: glm.fit: fitted probabilities numerically 0 or 1 occurred\n#> \n#> Run `show_notes(.Last.tune.result)` for more information.\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#player-effects", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#player-effects", + "title": "5 - Feature engineering", + "section": "Player effects ", + "text": "Player effects \n\nlibrary(embed)\n\nnhl_effect_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_lencode_mixed(shooter, goaltender, outcome = vars(on_goal)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-1", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-1", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nTune our glm_wflow.\nWhat happens if you don’t supply a grid argument to tune_grid()?\n\n\n\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#recipes-are-estimated", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#recipes-are-estimated", + "title": "5 - Feature engineering", + "section": "Recipes are estimated ", + "text": "Recipes are estimated \nPreprocessing steps in a recipe use the training set to compute quantities.\n\nWhat kind of quantities are computed for preprocessing?\n\nLevels of a factor\nWhether a column has zero variance\nNormalization\nFeature extraction\nEffect encodings\n\n\n\nWhen a recipe is part of a workflow, this estimation occurs when fit() is called." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#grid-results", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#grid-results", - "title": "6 - Tuning Hyperparameters", - "section": "Grid results ", - "text": "Grid results \n\nautoplot(glm_spline_res)" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#effect-encoding-results", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#effect-encoding-results", + "title": "5 - Feature engineering", + "section": "Effect encoding results ", + "text": "Effect encoding results \n\nnhl_effect_wflow <-\n nhl_glm_wflow %>%\n update_recipe(nhl_effect_rec)\n\nnhl_effect_res <-\n nhl_effect_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.791 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.805 1 NA Preprocessor1_Model1\n\nBetter and it can handle new players (if they occur)." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(glm_spline_res)\n#> # A tibble: 50 × 8\n#> angle coord_x .metric .estimator mean n std_err .config \n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 14 27 accuracy binary 0.804 1 NA Preprocessor01_Model1\n#> 2 14 27 roc_auc binary 0.815 1 NA Preprocessor01_Model1\n#> 3 4 42 accuracy binary 0.808 1 NA Preprocessor02_Model1\n#> 4 4 42 roc_auc binary 0.820 1 NA Preprocessor02_Model1\n#> 5 26 20 accuracy binary 0.805 1 NA Preprocessor03_Model1\n#> 6 26 20 roc_auc binary 0.819 1 NA Preprocessor03_Model1\n#> 7 36 16 accuracy binary 0.800 1 NA Preprocessor04_Model1\n#> 8 36 16 roc_auc binary 0.817 1 NA Preprocessor04_Model1\n#> 9 13 3 accuracy binary 0.803 1 NA Preprocessor05_Model1\n#> 10 13 3 roc_auc binary 0.807 1 NA Preprocessor05_Model1\n#> # ℹ 40 more rows" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#angle", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#angle", + "title": "5 - Feature engineering", + "section": "Angle", + "text": "Angle\n\nnhl_angle_rec <-\n nhl_effect_rec %>%\n step_mutate(\n angle = abs( atan2(abs(coord_y), (89 - coord_x) ) * (180 / pi) )\n )\n\n\n\nNote the danger of using step_mutate() – easy to have data leakage" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-1", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-1", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(glm_spline_res, summarize = FALSE)\n#> # A tibble: 50 × 7\n#> id angle coord_x .metric .estimator .estimate .config \n#> <chr> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 validation 14 27 accuracy binary 0.804 Preprocessor01_Model1\n#> 2 validation 14 27 roc_auc binary 0.815 Preprocessor01_Model1\n#> 3 validation 4 42 accuracy binary 0.808 Preprocessor02_Model1\n#> 4 validation 4 42 roc_auc binary 0.820 Preprocessor02_Model1\n#> 5 validation 26 20 accuracy binary 0.805 Preprocessor03_Model1\n#> 6 validation 26 20 roc_auc binary 0.819 Preprocessor03_Model1\n#> 7 validation 36 16 accuracy binary 0.800 Preprocessor04_Model1\n#> 8 validation 36 16 roc_auc binary 0.817 Preprocessor04_Model1\n#> 9 validation 13 3 accuracy binary 0.803 Preprocessor05_Model1\n#> 10 validation 13 3 roc_auc binary 0.807 Preprocessor05_Model1\n#> # ℹ 40 more rows" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#shot-from-the-defensive-zone", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#shot-from-the-defensive-zone", + "title": "5 - Feature engineering", + "section": "Shot from the defensive zone", + "text": "Shot from the defensive zone\n\nnhl_zone_rec <-\n nhl_angle_rec %>%\n step_mutate(\n defensive_zone = ifelse(coord_x <= -25.5, 1, 0)\n )" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choose-a-parameter-combination", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choose-a-parameter-combination", - "title": "6 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \n\nshow_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 8\n#> angle coord_x .metric .estimator mean n std_err .config \n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 4 42 roc_auc binary 0.820 1 NA Preprocessor02_Model1\n#> 2 26 20 roc_auc binary 0.819 1 NA Preprocessor03_Model1\n#> 3 36 16 roc_auc binary 0.817 1 NA Preprocessor04_Model1\n#> 4 40 44 roc_auc binary 0.817 1 NA Preprocessor08_Model1\n#> 5 37 27 roc_auc binary 0.816 1 NA Preprocessor15_Model1" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#behind-goal-line", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#behind-goal-line", + "title": "5 - Feature engineering", + "section": "Behind goal line", + "text": "Behind goal line\n\nnhl_behind_rec <-\n nhl_zone_rec %>%\n step_mutate(\n behind_goal_line = ifelse(coord_x >= 89, 1, 0)\n )" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "title": "6 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nselect_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 1 × 3\n#> angle coord_x .config \n#> <int> <int> <chr> \n#> 1 4 42 Preprocessor02_Model1\n\n\nThis best result has:\n\nlow-degree spline for angle (less “wiggly”, less complex)\nhigher-degree spline for coord_x (more “wiggly”, more complex)" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#fit-different-recipes", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#fit-different-recipes", + "title": "5 - Feature engineering", + "section": "Fit different recipes ", + "text": "Fit different recipes \nA workflow set can cross models and/or preprocessors and then resample them en masse.\n\nno_coord_rec <- \n nhl_indicators %>% \n step_rm(starts_with(\"coord\"))\n\nset.seed(9)\n\nnhl_glm_set_res <-\n workflow_set(\n list(`1_no_coord` = no_coord_rec, `2_other` = nhl_other_rec, \n `3_effects` = nhl_effect_rec, `4_angle` = nhl_angle_rec, \n `5_zone` = nhl_zone_rec, `6_bgl` = nhl_behind_rec),\n list(logistic = logistic_reg())\n ) %>%\n workflow_map(fn = \"fit_resamples\", resamples = nhl_val, verbose = TRUE, control = ctrl)" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-trees-1", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-trees-1", - "title": "6 - Tuning Hyperparameters", - "section": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\n\n\nReview how a decision tree model works:\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-4", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-4", + "title": "5 - Feature engineering", + "section": "Your turn", + "text": "Your turn\n\nCreate a workflow set with 2 or 3 recipes.\n(Consider using recipes we’ve already created.)\nUse workflow_map() to resample the workflow set.\n\n\n\n08:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#single-decision-tree", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#single-decision-tree", - "title": "6 - Tuning Hyperparameters", - "section": "Single decision tree", - "text": "Single decision tree" + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#compare-recipes", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#compare-recipes", + "title": "5 - Feature engineering", + "section": "Compare recipes", + "text": "Compare recipes\n\nlibrary(forcats)\ncollect_metrics(nhl_glm_set_res) %>%\n filter(.metric == \"roc_auc\") %>%\n mutate(\n features = gsub(\"_logistic\", \"\", wflow_id), \n features = fct_reorder(features, mean)\n ) %>%\n ggplot(aes(x = mean, y = features)) +\n geom_point(size = 3) +\n labs(y = NULL, x = \"ROC AUC (validation set)\")" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-trees-2", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-trees-2", - "title": "6 - Tuning Hyperparameters", - "section": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\nBoosting methods fit a sequence of tree-based models.\n\n\nEach tree is dependent on the one before and tries to compensate for any poor results in the previous trees.\nThis is like gradient-based steepest ascent methods from calculus." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#compare-recipes-1", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#compare-recipes-1", + "title": "5 - Feature engineering", + "section": "Compare recipes", + "text": "Compare recipes" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "title": "6 - Tuning Hyperparameters", - "section": "Boosted tree tuning parameters", - "text": "Boosted tree tuning parameters\nMost modern boosting methods have a lot of tuning parameters!\n\n\nFor tree growth and pruning (min_n, max_depth, etc)\nFor boosting (trees, stop_iter, learn_rate)\n\n\n\nWe’ll use early stopping to stop boosting when a few iterations produce consecutively worse results." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#debugging-a-recipe", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#debugging-a-recipe", + "title": "5 - Feature engineering", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. encoded_players) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#comparing-tree-ensembles", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#comparing-tree-ensembles", - "title": "6 - Tuning Hyperparameters", - "section": "Comparing tree ensembles", - "text": "Comparing tree ensembles\n\n\nRandom forest\n\nIndependent trees\nBootstrapped data\nNo pruning\n1000’s of trees\n\n\nBoosting\n\nDependent trees\nDifferent case weights\nTune tree parameters\nFar fewer trees\n\n\nThe general consensus for tree-based models is, in terms of performance: boosting > random forest > bagging > single trees." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#example", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#example", + "title": "5 - Feature engineering", + "section": "Example", + "text": "Example\n\nnhl_angle_fit <- prep(nhl_angle_rec)\n\ntidy(nhl_angle_fit, number = 1) %>% slice(1:4)\n#> # A tibble: 4 × 4\n#> level value terms id \n#> <chr> <dbl> <chr> <chr> \n#> 1 aaron_ekblad 0.250 shooter lencode_mixed_qBdjK\n#> 2 adam_clendening 0.202 shooter lencode_mixed_qBdjK\n#> 3 adam_cracknell 0.288 shooter lencode_mixed_qBdjK\n#> 4 adam_henrique 0.221 shooter lencode_mixed_qBdjK\n\nbake(nhl_angle_fit, nhl_train %>% slice(1:3), starts_with(\"coord\"), angle, shooter)\n#> # A tibble: 3 × 4\n#> coord_x coord_y angle shooter\n#> <dbl> <dbl> <dbl> <dbl>\n#> 1 67 -37 59.3 0.220\n#> 2 55 -7 11.6 0.302\n#> 3 42 -18 21.0 0.261" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-tree-code", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-tree-code", - "title": "6 - Tuning Hyperparameters", - "section": "Boosted tree code", - "text": "Boosted tree code\n\nxgb_spec <-\n boost_tree(\n trees = tune(), min_n = tune(), tree_depth = tune(),\n learn_rate = tune(), loss_reduction = tune()\n ) %>%\n set_mode(\"classification\") %>% \n set_engine(\"xgboost\") \n\nxgb_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_lencode_mixed(shooter, goaltender, outcome = vars(on_goal)) %>% \n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\nxgb_wflow <- \n workflow() %>% \n add_model(xgb_spec) %>% \n add_recipe(xgb_rec)\n\n\nvalidation is an argument to parsnip::xgb_train(), not directly to xgboost. It generates a validation set that is used by xgboost when evaluating model performance. It is eventually assigned to xgb.train(watchlist = list(validation = data)).\nSee translate(xgb_spec) to see where it is passed to parsnip::xgb_train()." + "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#more-on-recipes", + "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#more-on-recipes", + "title": "5 - Feature engineering", + "section": "More on recipes", + "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-2", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-2", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn", + "title": "3 - What makes a model?", "section": "Your turn", - "text": "Your turn\n\nCreate your boosted tree workflow.\n\n\n\n03:00" - }, - { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#running-in-parallel", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#running-in-parallel", - "title": "6 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" + "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n03:00\n\n\n\n\n\nlm for linear model\nglm for generalized linear model (e.g. logistic regression)\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#running-in-parallel-1", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#running-in-parallel-1", - "title": "6 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning ", - "text": "Tuning \nThis will take some time to run ⏳\n\nset.seed(9)\n\nxgb_res <-\n xgb_wflow %>%\n tune_grid(resamples = nhl_val, grid = 30, control = ctrl) # automatic grid now!" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-1", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-1", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlinear_reg()\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: lm\n\n\nModels have default engines" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-3", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-3", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nStart tuning the boosted tree model!\nWe won’t wait for everyone’s tuning to finish, but take this time to get it started before we move on.\n\n\n\n03:00" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-2", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-2", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-2", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-2", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\nxgb_res\n#> # Tuning results\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [5348/1338]> validation <tibble [60 × 9]> <tibble [0 × 3]> <tibble [40,140 × 11]>" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-3", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-3", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlinear_reg() %>%\n set_engine(\"glmnet\")\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: glmnet" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-3", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-3", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\nautoplot(xgb_res)" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-4", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-4", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlinear_reg() %>%\n set_engine(\"stan\")\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: stan" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#again-with-the-location-features", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#again-with-the-location-features", - "title": "6 - Tuning Hyperparameters", - "section": "Again with the location features", - "text": "Again with the location features\n\ncoord_rec <- \n xgb_rec %>%\n step_mutate(\n angle = abs( atan2(abs(coord_y), (89 - coord_x) ) * (180 / pi) ),\n defensive_zone = ifelse(coord_x <= -25.5, 1, 0),\n behind_goal_line = ifelse(coord_x >= 89, 1, 0)\n )\n\nxgb_coord_wflow <- \n workflow() %>% \n add_model(xgb_spec) %>% \n add_recipe(coord_rec)\n\nset.seed(9)\nxgb_coord_res <-\n xgb_coord_wflow %>%\n tune_grid(resamples = nhl_val, grid = 30, control = ctrl)" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-5", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-5", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#did-the-machine-figure-it-out", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#did-the-machine-figure-it-out", - "title": "6 - Tuning Hyperparameters", - "section": "Did the machine figure it out?", - "text": "Did the machine figure it out?\nNo extra features:\n\nshow_best(xgb_res, metric = \"roc_auc\", n = 3)\n#> # A tibble: 3 × 11\n#> trees min_n tree_depth learn_rate loss_reduction .metric .estimator mean n std_err .config \n#> <int> <int> <int> <dbl> <dbl> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1697 3 3 0.0116 0.674 roc_auc binary 0.804 1 NA Preprocessor1_Model02\n#> 2 1264 16 2 0.00732 0.000340 roc_auc binary 0.803 1 NA Preprocessor1_Model12\n#> 3 569 39 4 0.0272 0.0000288 roc_auc binary 0.800 1 NA Preprocessor1_Model30\n\nWith additional coordinate features:\n\nshow_best(xgb_coord_res, metric = \"roc_auc\", n = 3)\n#> # A tibble: 3 × 11\n#> trees min_n tree_depth learn_rate loss_reduction .metric .estimator mean n std_err .config \n#> <int> <int> <int> <dbl> <dbl> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1697 3 3 0.0116 0.674 roc_auc binary 0.807 1 NA Preprocessor1_Model02\n#> 2 1264 16 2 0.00732 0.000340 roc_auc binary 0.804 1 NA Preprocessor1_Model12\n#> 3 427 31 2 0.249 0.0000000773 roc_auc binary 0.803 1 NA Preprocessor1_Model23" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-6", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-6", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#compare-models", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#compare-models", - "title": "6 - Tuning Hyperparameters", - "section": "Compare models", - "text": "Compare models\nBest logistic regression results:\n\nglm_spline_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, .estimator, mean, n, std_err, .config)\n#> # A tibble: 1 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 roc_auc binary 0.820 1 NA Preprocessor02_Model1\n\n\nBest boosting results:\n\nxgb_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, .estimator, mean, n, std_err, .config)\n#> # A tibble: 1 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 roc_auc binary 0.804 1 NA Preprocessor1_Model02" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-7", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-7", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"regression\")\n#> Decision Tree Model Specification (regression)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#updating-the-workflow", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#updating-the-workflow", - "title": "6 - Tuning Hyperparameters", - "section": "Updating the workflow ", - "text": "Updating the workflow \n\nbest_auc <- select_best(glm_spline_res, metric = \"roc_auc\")\nbest_auc\n#> # A tibble: 1 × 3\n#> angle coord_x .config \n#> <int> <int> <chr> \n#> 1 4 42 Preprocessor02_Model1\n\nglm_spline_wflow <-\n glm_spline_wflow %>% \n finalize_workflow(best_auc)\n\nglm_spline_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: logistic_reg()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 7 Recipe Steps\n#> \n#> • step_lencode_mixed()\n#> • step_dummy()\n#> • step_mutate()\n#> • step_zv()\n#> • step_ns()\n#> • step_ns()\n#> • step_normalize()\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-8", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-8", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#the-final-fit-to-the-nhl-data", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#the-final-fit-to-the-nhl-data", - "title": "6 - Tuning Hyperparameters", - "section": "The final fit to the NHL data ", - "text": "The final fit to the NHL data \n\ntest_res <- \n glm_spline_wflow %>% \n last_fit(split = nhl_split)\n\ntest_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [6686/2229]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [2,229 × 6]> <workflow>\n\n\nRemember that last_fit() fits one time with the combined training and validation set, then evaluates one time with the testing set." + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-1", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-1", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code so it creates a different model.\n\n\n\n05:00\n\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-4", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-4", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nFinalize your workflow with the best parameters.\nCreate a final fit.\n\n\n\n08:00" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#models-well-be-using-today", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#models-well-be-using-today", + "title": "3 - What makes a model?", + "section": "Models we’ll be using today", + "text": "Models we’ll be using today\n\nLinear regression\nDecision trees" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#estimates-of-roc-auc", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#estimates-of-roc-auc", - "title": "6 - Tuning Hyperparameters", - "section": "Estimates of ROC AUC ", - "text": "Estimates of ROC AUC \nValidation results from tuning:\n\nglm_spline_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, mean, n, std_err)\n#> # A tibble: 1 × 4\n#> .metric mean n std_err\n#> <chr> <dbl> <int> <dbl>\n#> 1 roc_auc 0.820 1 NA\n\n\nTest set results:\n\ntest_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.800 Preprocessor1_Model1\n#> 2 roc_auc binary 0.807 Preprocessor1_Model1" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression", + "title": "3 - What makes a model?", + "section": "Linear regression", + "text": "Linear regression" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#final-fitted-workflow", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#final-fitted-workflow", - "title": "6 - Tuning Hyperparameters", - "section": "Final fitted workflow", - "text": "Final fitted workflow\nExtract the final fitted workflow, fit using the training set:\n\nfinal_glm_spline_wflow <- \n test_res %>% \n extract_workflow()\n\n# use this object to predict or deploy\npredict(final_glm_spline_wflow, nhl_test[1:3,])\n#> # A tibble: 3 × 1\n#> .pred_class\n#> <fct> \n#> 1 yes \n#> 2 yes \n#> 3 no" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression-1", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression-1", + "title": "3 - What makes a model?", + "section": "Linear regression", + "text": "Linear regression" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#next-steps", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#next-steps", - "title": "6 - Tuning Hyperparameters", - "section": "Next steps", - "text": "Next steps\n\nDocument the model.\n\n\n\nDeploy the model.\n\n\n\n\nCreate an applicability domain model to help monitor our data over time.\n\n\n\n\nUse explainers to characterize the model and the predictions." + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression-2", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression-2", + "title": "3 - What makes a model?", + "section": "Linear regression", + "text": "Linear regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nOutcome modeled as linear combination of predictors:\n\n\\(\\mbox{latency} = \\beta_0 + \\beta_1\\cdot\\mbox{age} + \\epsilon\\)\n\nFind a line that minimizes the mean squared error (MSE)" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-yourself", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-yourself", - "title": "6 - Tuning Hyperparameters", - "section": "Explain yourself", - "text": "Explain yourself\nThere are two categories of model explanations, global and local.\n\n\nGlobal model explanations provide an overall understanding aggregated over a whole set of observations.\nLocal model explanations provide information about a prediction for a single observation.\n\n\n\nYou can also build global model explanations by aggregating local model explanations." + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#a-tidymodels-explainer", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#a-tidymodels-explainer", - "title": "6 - Tuning Hyperparameters", - "section": "A tidymodels explainer", - "text": "A tidymodels explainer\nWe can build explainers using:\n\noriginal, basic predictors\nderived features\n\n\nlibrary(DALEXtra)\n\nglm_explainer <- explain_tidymodels(\n final_glm_spline_wflow,\n data = dplyr::select(nhl_train, -on_goal),\n # DALEX required an integer for factors:\n y = as.integer(nhl_train$on_goal) - 1,\n verbose = FALSE\n)" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees-1", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees-1", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates", - "title": "6 - Tuning Hyperparameters", - "section": "Explain the x coordinates", - "text": "Explain the x coordinates\nWith our explainer, let’s create partial dependence profiles:\n\nset.seed(123)\npdp_coord_x <- model_profile(\n glm_explainer,\n variables = \"coord_x\",\n N = 500,\n groups = \"strength\"\n)\n\n\nYou can use the default plot() method or create your own visualization." + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees-2", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees-2", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates-1", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates-1", - "title": "6 - Tuning Hyperparameters", - "section": "Explain the x coordinates", - "text": "Explain the x coordinates" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "title": "3 - What makes a model?", + "section": "All models are wrong, but some are useful!", + "text": "All models are wrong, but some are useful!\n\n\nLinear regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates-2", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates-2", - "title": "6 - Tuning Hyperparameters", - "section": "Explain the x coordinates", - "text": "Explain the x coordinates" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "title": "3 - What makes a model?", + "section": "Workflows bind preprocessors and models", + "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" }, { - "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-5", - "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-5", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nCreate an explainer for our glm model.\nTry using other variables, like extra_attacker or game_seconds.\n\n\n\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#what-is-wrong-with-this", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#what-is-wrong-with-this", + "title": "3 - What makes a model?", + "section": "What is wrong with this?", + "text": "What is wrong with this?" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 rsq standard 0.380\n#> 3 mae standard 40.2\n\n\n\nRMSE: difference between the predicted and observed values ⬇️\n\\(R^2\\): squared correlation between the predicted and observed values ⬆️\nMAE: similar to RMSE, but mean absolute error ⬇️" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#why-a-workflow", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#why-a-workflow", + "title": "3 - What makes a model?", + "section": "Why a workflow()? ", + "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering tomorrow!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-1", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-1", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-1", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\ntree_spec %>% \n fit(latency ~ ., data = frog_train) \n#> parsnip model object\n#> \n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-2", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-2", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n group_by(reflex) %>%\n rmse(latency, .pred)\n#> # A tibble: 3 × 4\n#> reflex .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 low rmse standard 94.3\n#> 2 mid rmse standard 101. \n#> 3 full rmse standard 51.2" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-2", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-2", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\nworkflow() %>%\n add_formula(latency ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = frog_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-3", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-3", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\nfrog_metrics <- metric_set(rmse, msd)\naugment(tree_fit, new_data = frog_test) %>%\n frog_metrics(latency, .pred)\n#> # A tibble: 2 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 msd standard -0.908" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-3", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-3", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\nworkflow(latency ~ ., tree_spec) %>% \n fit(data = frog_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-1", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-2", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-2", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code so it uses a linear model.\n\n\n\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-2", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-2", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#predict-with-your-model", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#predict-with-your-model", + "title": "3 - What makes a model?", + "section": "Predict with your model ", + "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\ntree_fit <-\n workflow(latency ~ ., tree_spec) %>% \n fit(data = frog_train)" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-3", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-3", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\ntree_fit %>%\n augment(frog_train)\n#> # A tibble: 456 × 6\n#> treatment reflex age t_o_d latency .pred\n#> <fct> <fct> <dbl> <fct> <dbl> <dbl>\n#> 1 control full 5.42 morning 33 39.8\n#> 2 control full 5.38 morning 19 66.7\n#> 3 control full 5.38 morning 2 66.7\n#> 4 control full 5.44 morning 39 39.8\n#> 5 control full 5.41 morning 42 39.8\n#> 6 control full 4.75 afternoon 20 59.8\n#> 7 control full 4.95 night 31 83.1\n#> 8 control full 5.42 morning 21 39.8\n#> 9 gentamicin full 5.39 morning 30 64.6\n#> 10 control full 4.55 afternoon 43 174. \n#> # ℹ 446 more rows\n\nWe call this “resubstitution” or “repredicting the training set”" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-3", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-3", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = frog_test)\nWhat do you get?\n\n\n\n03:00" + }, + { + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-4", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-4", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = frog_test)\nWhat do you get?\n\n\n\n03:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-4", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-4", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4\n\nWe call this a “resubstitution estimate”" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-5", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-5", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model-1", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model-1", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\nroundint = FALSE is only to quiet a warning" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-6", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-6", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4\n\n\n\ntree_fit %>%\n augment(frog_test) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model-2", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model-2", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn", - "title": "4 - Evaluating models", + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-5", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-5", + "title": "3 - What makes a model?", "section": "Your turn", - "text": "Your turn\n\nUse augment() and metrics() to compute a regression metric like mae().\nCompute the metrics for both training and testing data.\nNotice the evidence of overfitting! ⚠️\n\n\n\n05:00" + "text": "Your turn\n\nExtract the model engine object from your fitted linear workflow.\n⚠️ Never predict() with any extracted components!\n\n\n\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-7", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-7", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4 \n#> 2 rsq standard 0.494\n#> 3 mae standard 33.4\n\n\n\ntree_fit %>%\n augment(frog_test) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 rsq standard 0.380\n#> 3 mae standard 40.2\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#deploying-a-model", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#deploying-a-model", + "title": "3 - What makes a model?", + "section": "Deploying a model ", + "text": "Deploying a model \nHow do you use your new tree_fit model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"frog_hatching\")\nv\n#> \n#> ── frog_hatching ─ <bundled_workflow> model for deployment \n#> A rpart regression modeling workflow using 4 features\n\nLearn more at https://vetiver.rstudio.com" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#deploy-your-model-1", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#deploy-your-model-1", + "title": "3 - What makes a model?", + "section": "Deploy your model ", + "text": "Deploy your model \nHow do you use your new model tree_fit in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 3 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Users/emilhvitfeldt/Library/R/arm64/4.2/library/vetiver\n#> ├──/metadata (GET)\n#> ├──/ping (GET)\n#> └──/predict (POST)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-1", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-6", + "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-6", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-1", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n03:00" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#who-are-you", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-2", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-2", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(frog_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [410/46]> Fold01\n#> 2 <split [410/46]> Fold02\n#> 3 <split [410/46]> Fold03\n#> 4 <split [410/46]> Fold04\n#> 5 <split [410/46]> Fold05\n#> 6 <split [410/46]> Fold06\n#> 7 <split [411/45]> Fold07\n#> 8 <split [411/45]> Fold08\n#> 9 <split [411/45]> Fold09\n#> 10 <split [411/45]> Fold10" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#who-are-tidymodels", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-3", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-3", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWhat is in this?\n\nfrog_folds <- vfold_cv(frog_train)\nfrog_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n\n\nTalk about a list column, storing non-atomic types in dataframe" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#asking-for-help", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#asking-for-help", + "title": "1 - Introduction", + "section": "Asking for help", + "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-4", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-4", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(frog_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [364/92]> Fold1\n#> 2 <split [365/91]> Fold2\n#> 3 <split [365/91]> Fold3\n#> 4 <split [365/91]> Fold4\n#> 5 <split [365/91]> Fold5" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#section", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#section", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-5", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-5", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(frog_train, strata = latency)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [408/48]> Fold01\n#> 2 <split [408/48]> Fold02\n#> 3 <split [408/48]> Fold03\n#> 4 <split [409/47]> Fold04\n#> 5 <split [411/45]> Fold05\n#> 6 <split [412/44]> Fold06\n#> 7 <split [412/44]> Fold07\n#> 8 <split [412/44]> Fold08\n#> 9 <split [412/44]> Fold09\n#> 10 <split [412/44]> Fold10\n\n\nStratification often helps, with very little downside" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#tentative-plan-for-this-workshop", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#tentative-plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Tentative plan for this workshop", + "text": "Tentative plan for this workshop\n\nToday:\n\nYour data budget\nWhat makes a model\nEvaluating models\n\nTomorrow:\n\nFeature engineering\nTuning hyperparameters\nTransportation case study\nWrapping up!" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-6", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-6", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nfrog_folds <- vfold_cv(frog_train, v = 10, strata = latency)\nfrog_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [408/48]> Fold01\n#> 2 <split [408/48]> Fold02\n#> 3 <split [408/48]> Fold03\n#> 4 <split [409/47]> Fold04\n#> 5 <split [411/45]> Fold05\n#> 6 <split [412/44]> Fold06\n#> 7 <split [412/44]> Fold07\n#> 8 <split [412/44]> Fold08\n#> 9 <split [412/44]> Fold09\n#> 10 <split [412/44]> Fold10\n\n\nSet the seed when creating resamples" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#section-1", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#section-1", + "title": "1 - Introduction", + "section": "", + "text": "Introduce yourself to your neighbors 👋\n\n Log in to RStudio Cloud here (free):\nbit.ly/tidymodels-iceland-2022" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#fit-our-model-to-the-resamples", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#fit-our-model-to-the-resamples", - "title": "4 - Evaluating models", - "section": "Fit our model to the resamples", - "text": "Fit our model to the resamples\n\ntree_res <- fit_resamples(tree_wflow, frog_folds)\ntree_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]>" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\ntree_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 rmse standard 59.6 10 2.31 Preprocessor1_Model1\n#> 2 rsq standard 0.305 10 0.0342 Preprocessor1_Model1\n\n\nWe can reliably measure performance using only the training data 🎉" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning-1", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning-1", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#comparing-metrics", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#comparing-metrics", - "title": "4 - Evaluating models", - "section": "Comparing metrics ", - "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\ntree_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 rmse 59.6 10\n#> 2 rsq 0.305 10\n\n\nThe RMSE previously was\n\n49.36 for the training set\n59.16 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning-2", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning-2", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance-1", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_frog <- control_resamples(save_pred = TRUE)\ntree_res <- fit_resamples(tree_wflow, frog_folds, control = ctrl_frog)\n\ntree_preds <- collect_predictions(tree_res)\ntree_preds\n#> # A tibble: 456 × 5\n#> id .pred .row latency .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 Fold01 39.6 1 33 Preprocessor1_Model1\n#> 2 Fold01 72.1 3 2 Preprocessor1_Model1\n#> 3 Fold01 63.8 9 30 Preprocessor1_Model1\n#> 4 Fold01 72.1 13 46 Preprocessor1_Model1\n#> 5 Fold01 43.3 28 11 Preprocessor1_Model1\n#> 6 Fold01 61.7 35 41 Preprocessor1_Model1\n#> 7 Fold01 39.6 51 43 Preprocessor1_Model1\n#> 8 Fold01 134. 70 20 Preprocessor1_Model1\n#> 9 Fold01 70.6 74 21 Preprocessor1_Model1\n#> 10 Fold01 39.6 106 14 Preprocessor1_Model1\n#> # ℹ 446 more rows" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#your-turn", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#your-turn", + "title": "1 - Introduction", + "section": "Your turn", + "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-3", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-3", - "title": "4 - Evaluating models", - "section": "", - "text": "tree_preds %>% \n ggplot(aes(latency, .pred, color = id)) + \n geom_abline(lty = 2, col = \"gray\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-tidymodels", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-tidymodels", + "title": "1 - Introduction", + "section": "What is tidymodels? ", + "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.1.0 ──\n#> ✔ broom 1.0.5 ✔ rsample 1.1.1\n#> ✔ dials 1.2.0 ✔ tibble 3.2.1\n#> ✔ dplyr 1.1.2 ✔ tidyr 1.3.0\n#> ✔ infer 1.0.4 ✔ tune 1.1.1\n#> ✔ modeldata 1.1.0 ✔ workflows 1.1.3\n#> ✔ parsnip 1.1.0 ✔ workflowsets 1.0.1\n#> ✔ purrr 1.0.1 ✔ yardstick 1.2.0\n#> ✔ recipes 1.0.6\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Learn how to get started at https://www.tidymodels.org/start/" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#where-are-the-fitted-models", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#where-are-the-fitted-models", - "title": "4 - Evaluating models", - "section": "Where are the fitted models? ", - "text": "Where are the fitted models? \n\ntree_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [47 × 4]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [45 × 4]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n\n\n🗑️" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#the-whole-game", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#the-whole-game", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\nTomorrow we will walk through a case study in detail to illustrate feature engineering and model tuning.\nToday we will walk through the analysis at a higher level to show the model development process as a whole and give you an introduction to the data set.\nThe data are from the NHL where we want to predict whether a shot was on-goal or not! 🏒\nIt’s a good example to show how model development works." }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping", - "title": "4 - Evaluating models", - "section": "Bootstrapping", - "text": "Bootstrapping" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#shots-on-goal", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#shots-on-goal", + "title": "1 - Introduction", + "section": "Shots on goal", + "text": "Shots on goal" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping-1", - "title": "4 - Evaluating models", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(frog_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [456/163]> Bootstrap01\n#> 2 <split [456/166]> Bootstrap02\n#> 3 <split [456/173]> Bootstrap03\n#> 4 <split [456/177]> Bootstrap04\n#> 5 <split [456/166]> Bootstrap05\n#> 6 <split [456/163]> Bootstrap06\n#> 7 <split [456/164]> Bootstrap07\n#> 8 <split [456/165]> Bootstrap08\n#> 9 <split [456/170]> Bootstrap09\n#> 10 <split [456/177]> Bootstrap10\n#> # ℹ 15 more rows" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#data-spending", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#data-spending", + "title": "1 - Introduction", + "section": "Data spending", + "text": "Data spending" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-2", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-2", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCreate:\n\nbootstrap folds (change times from the default)\nvalidation set (use the reference guide to find the function)\n\nDon’t forget to set a seed when you resample!\n\n\n\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#a-first-model", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#a-first-model", + "title": "1 - Introduction", + "section": "A first model", + "text": "A first model" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping-2", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping-2", - "title": "4 - Evaluating models", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(322)\nbootstraps(frog_train, times = 10)\n#> # Bootstrap sampling \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [456/173]> Bootstrap01\n#> 2 <split [456/168]> Bootstrap02\n#> 3 <split [456/170]> Bootstrap03\n#> 4 <split [456/164]> Bootstrap04\n#> 5 <split [456/176]> Bootstrap05\n#> 6 <split [456/156]> Bootstrap06\n#> 7 <split [456/166]> Bootstrap07\n#> 8 <split [456/168]> Bootstrap08\n#> 9 <split [456/167]> Bootstrap09\n#> 10 <split [456/170]> Bootstrap10" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#starting-point-logistic-regression", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#starting-point-logistic-regression", + "title": "1 - Introduction", + "section": "Starting point: logistic regression", + "text": "Starting point: logistic regression\n\nWe’ll start by using basic logistic regression to predict our binary outcome.\nOur first model will have 12 simple predictor columns.\nOne initial question: there are 632 players taking shots.\nFor logistic regression, do we convert these to binary indicators (a.k.a. “dummies”)?" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#validation-set", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#validation-set", - "title": "4 - Evaluating models", - "section": "Validation set ", - "text": "Validation set \n\nset.seed(853)\nvalidation_split(frog_train, strata = latency)\n#> # Validation Set Split (0.75/0.25) using stratification \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [340/116]> validation\n\n\nA validation set is just another type of resample" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#basic-features-inc-dummy-variables", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#basic-features-inc-dummy-variables", + "title": "1 - Introduction", + "section": "Basic features (inc dummy variables)", + "text": "Basic features (inc dummy variables)" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#random-forest-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#random-forest-1", - "title": "4 - Evaluating models", - "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this tomorrow!), as long as there are enough trees" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#different-player-encoding", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#different-player-encoding", + "title": "1 - Introduction", + "section": "Different player encoding", + "text": "Different player encoding" + }, + { + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-about-location", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-about-location", + "title": "1 - Introduction", + "section": "What about location", + "text": "What about location\nThe previous models used the x/y coordinates.\nAre there better ways to represent shot location?\nHow can we make location more usable for the model?" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#create-a-random-forest-model", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#create-a-random-forest-model", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"regression\")\nrf_spec\n#> Random Forest Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-angle", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-angle", + "title": "1 - Introduction", + "section": "Add shot angle?", + "text": "Add shot angle?" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#create-a-random-forest-model-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#create-a-random-forest-model-1", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_wflow <- workflow(latency ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-from-defensive-zone", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-from-defensive-zone", + "title": "1 - Introduction", + "section": "Add shot from defensive zone?", + "text": "Add shot from defensive zone?" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-3", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-3", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\nplot true vs. predicted values\n\n\n\n\n08:00" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-behind-goal-line", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-behind-goal-line", + "title": "1 - Introduction", + "section": "Add shot behind goal line?", + "text": "Add shot behind goal line?" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance-2", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance-2", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nctrl_frog <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, frog_folds, control = ctrl_frog)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 rmse standard 55.9 10 1.76 Preprocessor1_Model1\n#> 2 rsq standard 0.372 10 0.0312 Preprocessor1_Model1" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#nonlinear-terms-for-angle-and-distance", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#nonlinear-terms-for-angle-and-distance", + "title": "1 - Introduction", + "section": "Nonlinear terms for angle and distance", + "text": "Nonlinear terms for angle and distance" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-5", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-5", - "title": "4 - Evaluating models", - "section": "", - "text": "collect_predictions(rf_res) %>% \n ggplot(aes(latency, .pred, color = id)) + \n geom_abline(lty = 2, col = \"gray\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#try-another-model", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#try-another-model", + "title": "1 - Introduction", + "section": "Try another model", + "text": "Try another model" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", - "title": "4 - Evaluating models", - "section": "How can we compare multiple model workflows at once?", - "text": "How can we compare multiple model workflows at once?" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#switch-to-boosting-and-basic-features", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#switch-to-boosting-and-basic-features", + "title": "1 - Introduction", + "section": "Switch to boosting and basic features", + "text": "Switch to boosting and basic features" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set", - "title": "4 - Evaluating models", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec))\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#boosting-with-location-features", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#boosting-with-location-features", + "title": "1 - Introduction", + "section": "Boosting with location features", + "text": "Boosting with location features" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set-1", - "title": "4 - Evaluating models", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = frog_folds)\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#choose-wisely", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#choose-wisely", + "title": "1 - Introduction", + "section": "Choose wisely…", + "text": "Choose wisely…" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set-2", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set-2", - "title": "4 - Evaluating models", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = frog_folds) %>%\n rank_results()\n#> # A tibble: 4 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_fo… Prepro… rmse 55.7 1.81 10 formula rand… 1\n#> 2 formula_rand_fo… Prepro… rsq 0.375 0.0309 10 formula rand… 1\n#> 3 formula_decisio… Prepro… rmse 59.6 2.31 10 formula deci… 2\n#> 4 formula_decisio… Prepro… rsq 0.305 0.0342 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#finalize-and-verify", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#finalize-and-verify", + "title": "1 - Introduction", + "section": "Finalize and verify", + "text": "Finalize and verify" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-4", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-4", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nWhen do you think a workflow set would be useful?\n\n\n\n03:00" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#and-so-on", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#and-so-on", + "title": "1 - Introduction", + "section": "… and so on", + "text": "… and so on\nOnce we find an acceptable model and feature set, the process is to\n\nConfirm our results on the test set.\nDocument the data and model development process.\nDeploy, monitor, etc." }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#the-final-fit", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#the-final-fit", - "title": "4 - Evaluating models", - "section": "The final fit ", - "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# frog_split has train + test info\nfinal_fit <- last_fit(rf_wflow, frog_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [456/116]> train/test split <tibble> <tibble> <tibble> <workflow>" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#lets-install-some-packages", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of RStudio Cloud:\n\ninstall.packages(c(\"Cubist\", \"DALEXtra\", \"doParallel\", \"earth\", \"embed\", \n \"forcats\", \"lme4\", \"parallelly\", \"ranger\", \"remotes\", \"rpart\", \n \"rpart.plot\", \"rules\", \"stacks\", \"tidymodels\",\n \"vetiver\", \"xgboost\"))\n\nremotes::install_github(\"topepo/ongoal@hockeyR\")\n\n\n\n Or log in to RStudio Cloud:\nbit.ly/tidymodels-iceland-2022" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 rmse standard 57.3 Preprocessor1_Model1\n#> 2 rsq standard 0.415 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" + "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#our-versions", + "href": "archive/2022-08-Reykjavik-City/01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nbroom (1.0.5, CRAN), DALEX (2.4.3, CRAN), DALEXtra (2.2.1, CRAN), dials (1.2.0, CRAN), doParallel (1.0.17, CRAN), dplyr (1.1.2, CRAN), embed (1.1.1, CRAN), ggplot2 (3.4.2, CRAN (R 4.2.1)), modeldata (1.1.0, CRAN), ongoal (0.0.4, Github (topepo/ongoal@2532b2404), parsnip (1.1.0, CRAN), purrr (1.0.1, CRAN), ranger (0.15.1, CRAN), recipes (1.0.6, CRAN), rpart (4.1.16, CRAN (R 4.2.1)), rpart.plot (3.1.1, CRAN), rsample (1.1.1, CRAN), scales (1.2.1, CRAN), stacks (1.0.2, CRAN), tibble (3.2.1, CRAN (R 4.2.1)), tidymodels (1.1.0, CRAN), tidyr (1.3.0, CRAN), tune (1.1.1, CRAN), vetiver (0.2.1, CRAN), workflows (1.1.3, CRAN), workflowsets (1.0.1, CRAN), xgboost (1.7.5.1, CRAN), and yardstick (1.2.0, CRAN)\nQuarto: 1.3.433" }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit-1", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit-1", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 116 × 5\n#> id .pred .row latency .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 train/test split 44.1 1 22 Preprocessor1_Model1\n#> 2 train/test split 101. 3 106 Preprocessor1_Model1\n#> 3 train/test split 75.7 6 39 Preprocessor1_Model1\n#> 4 train/test split 43.1 8 50 Preprocessor1_Model1\n#> 5 train/test split 43.5 10 63 Preprocessor1_Model1\n#> 6 train/test split 43.8 14 25 Preprocessor1_Model1\n#> 7 train/test split 51.2 16 48 Preprocessor1_Model1\n#> 8 train/test split 161. 17 91 Preprocessor1_Model1\n#> 9 train/test split 50.9 32 11 Preprocessor1_Model1\n#> 10 train/test split 177. 33 109 Preprocessor1_Model1\n#> # ℹ 106 more rows\n\n\nThese are predictions for the test set" + "objectID": "archive/2022-07-RStudio-conf/annotations.html", + "href": "archive/2022-07-RStudio-conf/annotations.html", + "title": "Annotations", + "section": "", + "text": "This page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources.\n\n\n\nThis is a pretty complex data usage scheme. That is mostly because of the validation set. In every other case, the situation is much more simple.\nThe important point here is that: tidymodels does most of this work for you. In other words, you don’t have to directly specify which data are being used where.\nIn a later section, we will talk about methods of resampling. These methods are like repeated validation sets. As an example, the popular 10-fold cross-validation method is one such type of resampling. Validation sets are special cases of resampling where there is a single “resample”.\nMost types of resampling use multiple hold-out sets of samples from the training set. In those cases, a diagram for data usage here would look like\n\n\n\n\n\n\n\n\n\nIn this case there is just “testing” and “training”. Once the final model is determined, the entire training set is used for the last fit.\nThis is the process that will be used for the tree frog data." }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-6", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-6", - "title": "4 - Evaluating models", + "objectID": "archive/2022-07-RStudio-conf/annotations.html#section", + "href": "archive/2022-07-RStudio-conf/annotations.html#section", + "title": "Annotations", "section": "", - "text": "collect_predictions(final_fit) %>%\n ggplot(aes(latency, .pred)) + \n geom_abline(lty = 2, col = \"deeppink4\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" + "text": "This page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit-2", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit-2", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1)) \n#> \n#> Type: Regression \n#> Number of trees: 1000 \n#> Sample size: 456 \n#> Number of independent variables: 4 \n#> Mtry: 2 \n#> Target node size: 5 \n#> Variable importance mode: none \n#> Splitrule: variance \n#> OOB prediction error (MSE): 3113.532 \n#> R squared (OOB): 0.355469\n\n\nUse this for prediction on new data, like for deploying" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#finalize-and-verify", + "href": "archive/2022-07-RStudio-conf/annotations.html#finalize-and-verify", + "title": "Annotations", + "section": "", + "text": "This is a pretty complex data usage scheme. That is mostly because of the validation set. In every other case, the situation is much more simple.\nThe important point here is that: tidymodels does most of this work for you. In other words, you don’t have to directly specify which data are being used where.\nIn a later section, we will talk about methods of resampling. These methods are like repeated validation sets. As an example, the popular 10-fold cross-validation method is one such type of resampling. Validation sets are special cases of resampling where there is a single “resample”.\nMost types of resampling use multiple hold-out sets of samples from the training set. In those cases, a diagram for data usage here would look like\n\n\n\n\n\n\n\n\n\nIn this case there is just “testing” and “training”. Once the final model is determined, the entire training set is used for the last fit.\nThis is the process that will be used for the tree frog data." }, { - "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-5", - "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-5", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nEnd of the day discussion!\nWhich model do you think you would decide to use?\nWhat surprised you the most?\nWhat is one thing you are looking forward to for tomorrow?\n\n\n\n05:00" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#data-splitting-and-spending", + "href": "archive/2022-07-RStudio-conf/annotations.html#data-splitting-and-spending", + "title": "Annotations", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n\n#> [1] 0.2655087 0.3721239 0.5728534\n\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching", - "title": "2 - Your data budget", - "section": "Data on tree frog hatching", - "text": "Data on tree frog hatching" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#what-is-wrong-with-this", + "href": "archive/2022-07-RStudio-conf/annotations.html#what-is-wrong-with-this", + "title": "Annotations", + "section": "What is wrong with this?", + "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-1", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-1", - "title": "2 - Your data budget", - "section": "Data on tree frog hatching", - "text": "Data on tree frog hatching\n\nRed-eyed tree frog embryos can hatch earlier than their normal ~7 days if they detect potential predator threat!\nType ?stacks::tree_frogs to learn more about this dataset, including references.\nWe are using a slightly modified version from stacks.\n\n\nlibrary(tidymodels)\n\ndata(\"tree_frogs\", package = \"stacks\")\ntree_frogs <- tree_frogs %>%\n mutate(t_o_d = factor(t_o_d),\n age = age / 86400) %>%\n filter(!is.na(latency)) %>%\n select(-c(clutch, hatched))" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#where-are-the-fitted-models", + "href": "archive/2022-07-RStudio-conf/annotations.html#where-are-the-fitted-models", + "title": "Annotations", + "section": "Where are the fitted models?", + "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-2", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-2", - "title": "2 - Your data budget", - "section": "Data on tree frog hatching", - "text": "Data on tree frog hatching\n\n\n\nN = 572\nA numeric outcome, latency\n4 other variables\n\ntreatment, reflex, and t_o_d are nominal predictors\nage is a numeric predictor\n\n\n\n\n\n\n\nlatency: How long it took the frog to hatch after being stimulated - i.e. after being poked by a blunt probe (in seconds).\ntreatment: Whether or not they got gentamicin, a compound that knocks out the embryo’s lateral line (a sensory organ).\nreflex: A measure of ear function (low, mid, full)\nt_o_d: Time that the stimulus was applied (morning, afternoon, night)\nage: Age at the time it was stimulated (in days)" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#the-final-fit", + "href": "archive/2022-07-RStudio-conf/annotations.html#the-final-fit", + "title": "Annotations", + "section": "The final fit", + "text": "The final fit\nSince our data spending scheme created the resamples from the training set, last_fit() will use all of the training data to fit the final workflow.\nAs shown in the Whole Game slides, there is a slightly different scheme used when we have a validation set (instead of multiple resamples like 10-fold CV)." }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-3", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-3", - "title": "2 - Your data budget", - "section": "Data on tree frog hatching", - "text": "Data on tree frog hatching\n\ntree_frogs\n#> # A tibble: 572 × 5\n#> treatment reflex age t_o_d latency\n#> <fct> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.40 morning 22\n#> 2 control low 4.18 night 360\n#> 3 control full 4.65 afternoon 106\n#> 4 control mid 4.14 night 180\n#> 5 control full 4.6 afternoon 60\n#> 6 gentamicin full 5.36 morning 39\n#> 7 control full 4.56 afternoon 214\n#> 8 control full 5.43 morning 50\n#> 9 control full 4.63 afternoon 224\n#> 10 control full 5.40 morning 63\n#> # ℹ 562 more rows" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#per-player-statistics", + "href": "archive/2022-07-RStudio-conf/annotations.html#per-player-statistics", + "title": "Annotations", + "section": "Per-player statistics", + "text": "Per-player statistics\nThe effect encoding method essentially takes the effect of a variable, like player, and makes a data column for that effect. In our example, the ability of a player to have an on-goal shot is quantified by a model and then added as a data column to be used in the model.\nSuppose NHL rookie Max has a single shot in the data and it was on goal. If we used a naive estimate for Max’s effect, the model is being told that Max should have a 100% chance of being on goal. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned player Davis, who has taken 250 shots and 75% of these were on goal. Davis’s proportion is more predictive because it is estimated with better data (i.e., more total shots). Partial pooling leverages the entire data set and can borrow strength from all of the players. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a player’s data is of good quality, the partial pooling effect estimate is closer to the raw proportion. Max’s data is not great and is “shrunk” towards the center of the overall on goal proportion. Since there is so little known about Max’s shot history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nIf the outcome were numeric, the effect would be the mean of the outcome per player. In this case, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." + "objectID": "archive/2022-07-RStudio-conf/annotations.html#player-effects", + "href": "archive/2022-07-RStudio-conf/annotations.html#player-effects", + "title": "Annotations", + "section": "Player effects", + "text": "Player effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for players with small sample sizes. It can’t correct for improper data usage or data leakage though." }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-1", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-1", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#angle", + "href": "archive/2022-07-RStudio-conf/annotations.html#angle", + "title": "Annotations", + "section": "Angle", + "text": "Angle\nAbout these formulas…\nThe coordinates for the rink are centered at (0, 0) and the goal lines are both 89 ft from center. The center of the goal on the left is at (-89, 0) and the right-hand goal is centered at (89, 0).\nFor the distance to center ice, the formula is\n\\[d_{center} = \\sqrt{x^2 + y^2}\\]\nWe want distance to the goal line(s), so we first use x* = (89 - abs(coord_x)) to make the side of the rink irrelevant and then compute\n\\[d_{goal} = \\sqrt{x^{*2} + y^2}\\]\nIn the code, we log the distance value to help its distribution become more symmetric.\nFor angle to center, the formula is\n\\[a = \\tan^{-1}\\left(\\frac{y}{x}\\right)\\]\nThis is in radian units and we can convert to degrees using\n\\[a = \\frac{180}{\\pi}\\tan^{-1}\\left(\\frac{y}{x}\\right)\\]\nFor the angle to the goal, we need to alter \\(x\\) and \\(y\\) again. We’ll use \\(x^*\\) from above and also use the absolute value of \\(y\\) so that the degrees range from 0 to 180." }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-2", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-2", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." + "objectID": "archive/2022-07-RStudio-conf/annotations.html#update-parameter-ranges", + "href": "archive/2022-07-RStudio-conf/annotations.html#update-parameter-ranges", + "title": "Annotations", + "section": "Update parameter ranges", + "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nIn our case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_ns(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n\n#> # A tibble: 1 × 5\n#> name call_info source component component_id\n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_ns ns_P1Tjg\n\nns_tunable$call_info\n\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 1 15" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n03:00" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#spline-grid-search", + "href": "archive/2022-07-RStudio-conf/annotations.html#spline-grid-search", + "title": "Annotations", + "section": "Spline grid search", + "text": "Spline grid search\nWhat’s going on with the\n\nprediction from a rank-deficient fit may be misleading\n\nwarnings?\nFor linear regression, a computation is used called matrix inversion. The matrix in question is called the “model matrix” and it contains the predictor set for the training data.\nMatrix inversion can fail if two or more columns:\n\nare identical, or\nadd up to some other column.\n\nThese situations are called linear dependencies.\nWhen this happens, lm() is pretty tolerant. It does not fail but does not compute regression coefficients for a minimal number of predictors involved in the dependency (and issues the warning above).\nFor these data, there are three dependencies between:\n\ndefense_team_PIT and offense_team_PIT\nstrength_short_handed, player_diff, and strength_power_play\nyear, month_Oct, month_Nov, and month_Dec\n\nThe first one is easy to explain. For each row, when one these two PIT column has a one, the other must have a zero. The linear regression intercept is represented in the model matrix as a column of all ones. The dependency is\n(Intercept) = defense_team_PIT + offense_team_PIT\nThe way to avoid this problem is to use step_lincomb(all_numeric_predictors()) in the recipe. This step removes the minimum number of columns to avoid the issue.\ntl;dr\nLinear regression detects some redundancies in the predictor set. We can ignore the warnings since lm() can deal with it or use step_lincomb() to avoid the warnings." }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-3", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-3", - "title": "2 - Your data budget", - "section": "Data splitting and spending ", - "text": "Data splitting and spending \n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs)\nfrog_split\n#> <Training/Testing/Total>\n#> <429/143/572>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#boosted-tree-tuning-parameters", + "href": "archive/2022-07-RStudio-conf/annotations.html#boosted-tree-tuning-parameters", + "title": "Annotations", + "section": "Boosted tree tuning parameters", + "text": "Boosted tree tuning parameters\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time." + }, + { + "objectID": "archive/2022-07-RStudio-conf/annotations.html#boosted-tree-code", + "href": "archive/2022-07-RStudio-conf/annotations.html#boosted-tree-code", + "title": "Annotations", + "section": "Boosted tree code", + "text": "Boosted tree code\nWe set an engine argument called validation here. That’s not an argument to any function in the xgboost package.\nparsnip has its own wrapper around (xgboost::xgb.train()) called xgb_train(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the xgboost entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for xgboost)." }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#accessing-the-data", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#accessing-the-data", - "title": "2 - Your data budget", - "section": "Accessing the data ", - "text": "Accessing the data \n\nfrog_train <- training(frog_split)\nfrog_test <- testing(frog_split)" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#the-final-fit-to-the-nhl-data", + "href": "archive/2022-07-RStudio-conf/annotations.html#the-final-fit-to-the-nhl-data", + "title": "Annotations", + "section": "The final fit to the NHL data", + "text": "The final fit to the NHL data\nRecall that last_fit() uses the objects produced by initial_split() to determine what data are used for the final model fit and which are used as the test set.\nFor the validation set, last_fit() will use the non-testing data to create the final model fit. This includes the training and validation set.\nThere is no agreement in the community on whether this is the best approach or if we should just use the training set. There are good arguments either way.\nIf you only want to use the training set for the final model, you can do this via:\n\ntraining_data <- nhl_val$splits[[1]] %>% analysis()\n\n# Use `fit()` to train the model on just the training set\nfinal_glm_spline_wflow <- \n glm_spline_wflow %>% \n fit(data = training_data)\n\n# Create test set predictions\ntest_set_pred <- augment(final_glm_spline_wflow, nhl_test)\n\n# Setup and compute the test set metrics\ncls_metrics <- metric_set(roc_auc, accuracy)\n\ntest_res <- \n test_set_pred %>% \n cls_metrics(on_goal, estimate = .pred_class, .pred_yes)\ntest_res" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#the-training-set", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#the-training-set", - "title": "2 - Your data budget", - "section": "The training set", - "text": "The training set\n\nfrog_train\n#> # A tibble: 429 × 5\n#> treatment reflex age t_o_d latency\n#> <fct> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.36 morning 36\n#> 2 gentamicin full 5.37 morning 72\n#> 3 gentamicin full 4.65 afternoon 141\n#> 4 control full 5.42 morning 27\n#> 5 control full 5.43 morning 27\n#> 6 gentamicin full 5.38 morning 73\n#> 7 gentamicin full 5.42 morning 68\n#> 8 gentamicin full 4.75 afternoon 124\n#> 9 control full 5.00 night 62\n#> 10 control full 5.39 morning 25\n#> # ℹ 419 more rows" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#explain-yourself", + "href": "archive/2022-07-RStudio-conf/annotations.html#explain-yourself", + "title": "Annotations", + "section": "Explain yourself", + "text": "Explain yourself\nSome other resources:\n\nTMwR chapter Explaining Models and Predictions\nExplanatory Model Analysis book\nInterpretable Machine Learning book\nDefinitions, methods, and applications in interpretable machine learning" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#the-test-set", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#the-test-set", - "title": "2 - Your data budget", - "section": "The test set ", - "text": "The test set \n\nfrog_test\n#> # A tibble: 143 × 5\n#> treatment reflex age t_o_d latency\n#> <fct> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.40 morning 22\n#> 2 control low 4.18 night 360\n#> 3 control full 4.63 afternoon 224\n#> 4 gentamicin full 4.75 afternoon 158\n#> 5 control mid 4.22 night 91\n#> 6 gentamicin full 4.89 night 301\n#> 7 control full 5.38 morning 2\n#> 8 control full 4.80 afternoon 56\n#> 9 control full 5.36 morning 11\n#> 10 control full 5.40 morning 64\n#> # ℹ 133 more rows" + "objectID": "archive/2022-07-RStudio-conf/annotations.html#a-tidymodels-explainer", + "href": "archive/2022-07-RStudio-conf/annotations.html#a-tidymodels-explainer", + "title": "Annotations", + "section": "A tidymodels explainer", + "text": "A tidymodels explainer\nFor our example, the angle was an original predictor. Recall that we made spline terms from this predictor, so there are derived features such as angle_ns_1 and so on.\nOriginal versus derived doesn’t affect local explainers since we are focused on a single prediction.\nFor global explainers, we should decide between:\n\nexplaining the overall affect of angle (lumping all its features into one importance score), or\nexplaining the effect of each term in the model (including angle_ns_1 and so on).\n\nThe choice depends on what you want. For example, if we have an original date predictor and make features for month and year, is it more informative to know if date is important (overall) or exactly how the date is important? You might want to look at it both ways." }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn-1", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn-1", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n05:00" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-parameters", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-parameters", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-4", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-4", - "title": "2 - Your data budget", - "section": "Data splitting and spending ", - "text": "Data splitting and spending \n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs, prop = 0.8)\nfrog_train <- training(frog_split)\nfrog_test <- testing(frog_split)\n\nnrow(frog_train)\n#> [1] 457\nnrow(frog_test)\n#> [1] 115" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#optimize-tuning-parameters", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#optimize-tuning-parameters", + "title": "6 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-1", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-1", - "title": "2 - Your data budget", - "section": "", - "text": "We will use this tomorrow" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "title": "6 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn-2", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn-2", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nExplore the frog_train data on your own!\n\nWhat’s the distribution of the outcome, latency?\nWhat’s the distribution of numeric variables like age?\nHow does latency differ across the categorical variables?\n\n\n\n\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choosing-tuning-parameters", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choosing-tuning-parameters", + "title": "6 - Tuning Hyperparameters", + "section": "Choosing tuning parameters ", + "text": "Choosing tuning parameters \nLet’s take our previous recipe and add a few changes:\n\nglm_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_lencode_mixed(player, outcome = vars(on_goal)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_mutate(\n angle = abs(atan2(abs(coord_y), (89 - abs(coord_x))) * (180 / pi)),\n distance = sqrt((89 - abs(coord_x))^2 + abs(coord_y)^2),\n distance = log(distance),\n behind_goal_line = ifelse(abs(coord_x) >= 89, 1, 0)\n ) %>%\n step_rm(coord_x, coord_y) %>%\n step_zv(all_predictors()) %>%\n step_ns(angle, deg_free = tune(\"angle\")) %>%\n step_ns(distance, deg_free = tune(\"distance\")) %>%\n step_normalize(all_numeric_predictors())\n\n\nLet’s tune() our spline terms!" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-3", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-3", - "title": "2 - Your data budget", - "section": "", - "text": "ggplot(frog_train, aes(latency)) +\n geom_histogram(bins = 20)\n\n\n\nThis histogram brings up a concern. What if in our training set we get unlucky and sample few or none of these large values? That could mean that our model wouldn’t be able to predict such values. Let’s come back to that!" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choosing-tuning-parameters-1", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choosing-tuning-parameters-1", + "title": "6 - Tuning Hyperparameters", + "section": "Choosing tuning parameters ", + "text": "Choosing tuning parameters \nLet’s take our previous recipe and add a few changes:\n\nglm_spline_wflow <-\n workflow() %>%\n add_model(logistic_reg()) %>%\n add_recipe(glm_rec)" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-4", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-4", - "title": "2 - Your data budget", + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#section", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#section", + "title": "6 - Tuning Hyperparameters", "section": "", - "text": "ggplot(frog_train, aes(latency, treatment, fill = treatment)) +\n geom_boxplot(alpha = 0.5, show.legend = FALSE)" + "text": "Splines replace the existing numeric predictor with a set of columns that allow a model to emulate a flexible, nonlinear relationship.\nMore spline terms = more “wiggly”, i.e. flexibly model a nonlinear relationship\nHow many spline terms? This is called degrees of freedom\n2 and 5 look like they underfit; 20 and 100 look like they overfit" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-5", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-5", - "title": "2 - Your data budget", - "section": "", - "text": "frog_train %>%\n ggplot(aes(latency, reflex, fill = reflex)) +\n geom_boxplot(alpha = 0.3, show.legend = FALSE)" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#splines-and-nonlinear-relationships", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#splines-and-nonlinear-relationships", + "title": "6 - Tuning Hyperparameters", + "section": "Splines and nonlinear relationships", + "text": "Splines and nonlinear relationships\n\n\nOur hockey data exhibits nonlinear relationships\nWe can model nonlinearity like this via a model (later this afternoon) or feature engineering\nHow do we decide how “wiggly” or flexible to make our spline features? TUNING" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-6", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-6", - "title": "2 - Your data budget", - "section": "", - "text": "ggplot(frog_train, aes(age, latency, color = reflex)) +\n geom_point(alpha = .8, size = 2)" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#grid-search", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#grid-search", + "title": "6 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nParameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-7", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-7", - "title": "2 - Your data budget", - "section": "", - "text": "Stratified sampling would split within each quartile\n\nBased on our exploration, we realized that stratifying by latency might help get a consistent distribution. For instance, we’d include high and low latency in both the test and training" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid", + "title": "6 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\nglm_spline_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 2 parameters for tuning\n#> \n#> identifier type object\n#> angle deg_free nparam[+]\n#> distance deg_free nparam[+]\n\n\nA parameter set can be updated (e.g. to change the ranges)." }, { - "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#stratification", - "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#stratification", - "title": "2 - Your data budget", - "section": "Stratification", - "text": "Stratification\nUse strata = latency\n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs, prop = 0.8, strata = latency)\nfrog_split\n#> <Training/Testing/Total>\n#> <456/116/572>\n\n\nStratification often helps, with very little downside" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid-1", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid-1", + "title": "6 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\n\n\nset.seed(2)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 23 × 2\n#> angle distance\n#> <int> <int>\n#> 1 12 4\n#> 2 15 8\n#> 3 6 14\n#> 4 10 5\n#> 5 12 12\n#> 6 7 8\n#> 7 14 3\n#> 8 14 13\n#> 9 11 12\n#> 10 8 11\n#> # … with 13 more rows\n#> # ℹ Use `print(n = ...)` to see more rows\n\n\n\n\nA space-filling design like this tends to perform better than random grids.\nSpace-filling designs are also usually more efficient than regular grids." }, { - "objectID": "archive/2022-07-RStudio-conf/index.html", - "href": "archive/2022-07-RStudio-conf/index.html", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels presented at the 2022 RStudio conference. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n03:00" }, { - "objectID": "archive/2022-07-RStudio-conf/index.html#welcome", - "href": "archive/2022-07-RStudio-conf/index.html#welcome", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels presented at the 2022 RStudio conference. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid-2", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid-2", + "title": "6 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\nset.seed(2)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 25)\n\ngrid\n#> # A tibble: 225 × 2\n#> angle distance\n#> <int> <int>\n#> 1 1 1\n#> 2 2 1\n#> 3 3 1\n#> 4 4 1\n#> 5 5 1\n#> 6 6 1\n#> 7 7 1\n#> 8 8 1\n#> 9 9 1\n#> 10 10 1\n#> # … with 215 more rows\n#> # ℹ Use `print(n = ...)` to see more rows\n\n\nNote that even though we requested 25x25=625 rows, we only got 15x15=225 back, since the deg_free parameters only have a range of 1->15." }, { - "objectID": "archive/2022-07-RStudio-conf/index.html#is-this-workshop-for-me", - "href": "archive/2022-07-RStudio-conf/index.html#is-this-workshop-for-me", - "title": "Machine learning with tidymodels", - "section": "Is this workshop for me? ", - "text": "Is this workshop for me? \nThis course assumes intermediate R knowledge. This workshop is for you if:\n\nYou can use the magrittr pipe %>% and/or native pipe |>\nYou are familiar with functions from dplyr, tidyr, and ggplot2\nYou can read data into R, transform and reshape data, and make a wide variety of graphs\n\nWe expect participants to have some exposure to basic statistical concepts, but NOT intermediate or expert familiarity with modeling or machine learning." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#update-parameter-ranges", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#update-parameter-ranges", + "title": "6 - Tuning Hyperparameters", + "section": "Update parameter ranges ", + "text": "Update parameter ranges \n\nset.seed(2)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n update(angle = spline_degree(c(2L, 20L)),\n distance = spline_degree(c(2L, 20L))) %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 24 × 2\n#> angle distance\n#> <int> <int>\n#> 1 16 6\n#> 2 20 11\n#> 3 8 19\n#> 4 14 7\n#> 5 16 17\n#> 6 10 11\n#> 7 19 5\n#> 8 18 17\n#> 9 15 16\n#> 10 11 15\n#> # … with 14 more rows\n#> # ℹ Use `print(n = ...)` to see more rows\n\n\nEven though angle is a deg_free parameter in step_ns(), we don’t use the dials deg_free() object here. We have a special spline_degree() function that has better defaults for splines." }, { - "objectID": "archive/2022-07-RStudio-conf/index.html#preparation", - "href": "archive/2022-07-RStudio-conf/index.html#preparation", - "title": "Machine learning with tidymodels", - "section": "Preparation", - "text": "Preparation\nPlease join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2022.02), available at https://www.rstudio.com/download\nThe following R packages, which you can install from the R console:\n\n\ninstall.packages(c(\"DALEXtra\", \"doParallel\", \"embed\", \"forcats\",\n \"lme4\", \"ranger\", \"remotes\", \"rpart\", \n \"rpart.plot\", \"stacks\", \"tidymodels\",\n \"vetiver\", \"xgboost\"))\n\nremotes::install_github(\"topepo/ongoal@v0.0.2\")" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#the-results", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#the-results", + "title": "6 - Tuning Hyperparameters", + "section": "The results ", + "text": "The results \n\n\ngrid %>% \n ggplot(aes(angle, distance)) +\n geom_point(size = 4)" }, { - "objectID": "archive/2022-07-RStudio-conf/index.html#slides", - "href": "archive/2022-07-RStudio-conf/index.html#slides", - "title": "Machine learning with tidymodels", - "section": "Slides", - "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. There are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nDay One\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n\n\n\nDay Two\n\n05: Feature engineering\n06: Tuning hyperparameters\n07: Wrapping up\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#spline-grid-search", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#spline-grid-search", + "title": "6 - Tuning Hyperparameters", + "section": "Spline grid search ", + "text": "Spline grid search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE, parallel_over = \"everything\")\n\nglm_spline_res <-\n glm_spline_wflow %>%\n tune_grid(resamples = nhl_val, grid = grid, control = ctrl)\n\nglm_spline_res\n#> # Tuning results\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7288/1822]> validation <tibble [48 × 6]> <tibble [24 × 3]> <tibble [43,728 × 8]>\n#> \n#> There were issues with some computations:\n#> \n#> - Warning(s) x24: prediction from a rank-deficient fit may be misleading\n#> \n#> Run `show_notes(.Last.tune.result)` for more information.\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" }, { - "objectID": "archive/2022-07-RStudio-conf/index.html#code", - "href": "archive/2022-07-RStudio-conf/index.html#code", - "title": "Machine learning with tidymodels", - "section": "Code", - "text": "Code\nQuarto files for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-1", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-1", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nTune our glm_wflow.\nWhat happens if you don’t supply a grid argument to tune_grid()?\n\n\n\n05:00" }, { - "objectID": "archive/2022-07-RStudio-conf/index.html#acknowledgments", - "href": "archive/2022-07-RStudio-conf/index.html#acknowledgments", - "title": "Machine learning with tidymodels", - "section": "Acknowledgments", - "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#grid-results", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#grid-results", + "title": "6 - Tuning Hyperparameters", + "section": "Grid results ", + "text": "Grid results \n\nautoplot(glm_spline_res)" }, { - "objectID": "archive/2022-07-RStudio-conf/index.html#reuse-and-licensing", - "href": "archive/2022-07-RStudio-conf/index.html#reuse-and-licensing", - "title": "Machine learning with tidymodels", - "section": "Reuse and licensing", - "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(glm_spline_res)\n#> # A tibble: 48 × 8\n#> angle distance .metric .estimator mean n std_err .config \n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 16 6 accuracy binary 0.610 1 NA Preprocessor01_Model1\n#> 2 16 6 roc_auc binary 0.649 1 NA Preprocessor01_Model1\n#> 3 20 11 accuracy binary 0.613 1 NA Preprocessor02_Model1\n#> 4 20 11 roc_auc binary 0.649 1 NA Preprocessor02_Model1\n#> 5 8 19 accuracy binary 0.619 1 NA Preprocessor03_Model1\n#> 6 8 19 roc_auc binary 0.652 1 NA Preprocessor03_Model1\n#> 7 14 7 accuracy binary 0.610 1 NA Preprocessor04_Model1\n#> 8 14 7 roc_auc binary 0.650 1 NA Preprocessor04_Model1\n#> 9 16 17 accuracy binary 0.617 1 NA Preprocessor05_Model1\n#> 10 16 17 roc_auc binary 0.649 1 NA Preprocessor05_Model1\n#> # … with 38 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/07-wrapping-up.html#your-turn", - "href": "archive/2022-07-RStudio-conf/07-wrapping-up.html#your-turn", - "title": "7 - Wrapping up", - "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n05:00" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-1", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-1", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(glm_spline_res, summarize = FALSE)\n#> # A tibble: 48 × 7\n#> id angle distance .metric .estimator .estimate .config \n#> <chr> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 validation 16 6 accuracy binary 0.610 Preprocessor01_Model1\n#> 2 validation 16 6 roc_auc binary 0.649 Preprocessor01_Model1\n#> 3 validation 20 11 accuracy binary 0.613 Preprocessor02_Model1\n#> 4 validation 20 11 roc_auc binary 0.649 Preprocessor02_Model1\n#> 5 validation 8 19 accuracy binary 0.619 Preprocessor03_Model1\n#> 6 validation 8 19 roc_auc binary 0.652 Preprocessor03_Model1\n#> 7 validation 14 7 accuracy binary 0.610 Preprocessor04_Model1\n#> 8 validation 14 7 roc_auc binary 0.650 Preprocessor04_Model1\n#> 9 validation 16 17 accuracy binary 0.617 Preprocessor05_Model1\n#> 10 validation 16 17 roc_auc binary 0.649 Preprocessor05_Model1\n#> # … with 38 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/07-wrapping-up.html#resources-to-keep-learning", - "href": "archive/2022-07-RStudio-conf/07-wrapping-up.html#resources-to-keep-learning", - "title": "7 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Twitter and at the tidyverse blog for updates!" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination", + "title": "6 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \n\nshow_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 8\n#> angle distance .metric .estimator mean n std_err .config \n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 5 8 roc_auc binary 0.653 1 NA Preprocessor12_Model1\n#> 2 6 9 roc_auc binary 0.653 1 NA Preprocessor17_Model1\n#> 3 3 15 roc_auc binary 0.653 1 NA Preprocessor13_Model1\n#> 4 3 13 roc_auc binary 0.652 1 NA Preprocessor11_Model1\n#> 5 5 19 roc_auc binary 0.652 1 NA Preprocessor24_Model1" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#working-with-our-predictors", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#working-with-our-predictors", - "title": "5 - Feature engineering", - "section": "Working with our predictors", - "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for lm()).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "title": "6 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nselect_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 1 × 3\n#> angle distance .config \n#> <int> <int> <chr> \n#> 1 5 8 Preprocessor12_Model1\n\n\nThis best result has:\n\nlow-degree spline for angle (less “wiggly”, less complex)\nhigher-degree spline for distance (more “wiggly”, more complex)" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-is-feature-engineering", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-is-feature-engineering", - "title": "5 - Feature engineering", - "section": "What is feature engineering?", - "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPCA feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-2", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-2", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nTry an alternative selection strategy.\nRead the docs for select_by_pct_loss().\nTry choosing a model that has a simpler (less “wiggly”) relationship for distance.\n\n\n\n05:00" + }, + { + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination-2", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination-2", + "title": "6 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \n\nselect_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 1 × 3\n#> angle distance .config \n#> <int> <int> <chr> \n#> 1 5 8 Preprocessor12_Model1\nselect_by_pct_loss(glm_spline_res, distance, metric = \"roc_auc\")\n#> # A tibble: 1 × 10\n#> angle distance .metric .estimator mean n std_err .config .best .loss\n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> <dbl> <dbl>\n#> 1 13 4 roc_auc binary 0.646 1 NA Preprocessor20_Model1 0.653 0.984" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#example-dates", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#example-dates", - "title": "5 - Feature engineering", - "section": "Example: Dates", - "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen a date column is used in its native format, it is usually converted by an R model to an integer.\n\n\nIt can be re-engineered as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-trees-1", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-trees-1", + "title": "6 - Tuning Hyperparameters", + "section": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\n\n\nReview how a decision tree model works:\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#general-definitions", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#general-definitions", - "title": "5 - Feature engineering", - "section": "General definitions ", - "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\nIn a little bit, we’ll see successful (and unsuccessful) feature engineering methods for our example data.\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#single-decision-tree", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#single-decision-tree", + "title": "6 - Tuning Hyperparameters", + "section": "Single decision tree", + "text": "Single decision tree" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#the-nhl-data", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#the-nhl-data", - "title": "5 - Feature engineering", - "section": "The NHL data 🏒", - "text": "The NHL data 🏒\n\nFrom Pittsburgh Penguins games, 12,147 shots\nData from the 2015-2016 season\n\n\nLet’s predict whether a shot is on-goal (a goal or blocked by goaltender) or not." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-trees-2", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-trees-2", + "title": "6 - Tuning Hyperparameters", + "section": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\nBoosting methods fit a sequence of tree-based models.\n\n\nEach tree is dependent on the one before and tries to compensate for any poor results in the previous trees.\nThis is like gradient-based steepest ascent methods from calculus." }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#case-study", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#case-study", - "title": "5 - Feature engineering", - "section": "Case study", - "text": "Case study\n\nlibrary(tidymodels)\nlibrary(ongoal)\n\ntidymodels_prefer()\n\nglimpse(season_2015)\n#> Rows: 12,147\n#> Columns: 17\n#> $ on_goal <fct> yes, no, no, yes, no, no, yes, no, yes, no, no, no, …\n#> $ period <int> 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1…\n#> $ period_type <fct> regular, regular, regular, regular, regular, regular…\n#> $ coord_x <dbl> -53, 68, -42, -77, -67, 55, 77, 62, 59, 76, 44, 62, …\n#> $ coord_y <dbl> -18, -12, -18, 9, -5, -12, 13, 14, -5, -6, 7, -2, -2…\n#> $ game_time <dbl> 0.300000, 0.900000, 1.250000, 1.783333, 2.050000, 3.…\n#> $ strength <fct> even, even, even, even, even, even, even, even, even…\n#> $ player <fct> victor_hedman, evgeni_malkin, jason_garrison, ondrej…\n#> $ player_diff <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ offense_team <fct> TBL, PIT, TBL, TBL, TBL, PIT, PIT, PIT, PIT, PIT, PI…\n#> $ defense_team <fct> PIT, TBL, PIT, PIT, PIT, TBL, TBL, TBL, TBL, TBL, TB…\n#> $ offense_goal_diff <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ game_type <fct> playoff, playoff, playoff, playoff, playoff, playoff…\n#> $ position <fct> defenseman, center, defenseman, left_wing, defensema…\n#> $ dow <fct> Sat, Sat, Sat, Sat, Sat, Sat, Sat, Sat, Sat, Sat, Sa…\n#> $ month <fct> May, May, May, May, May, May, May, May, May, May, Ma…\n#> $ year <dbl> 2016, 2016, 2016, 2016, 2016, 2016, 2016, 2016, 2016…" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "title": "6 - Tuning Hyperparameters", + "section": "Boosted tree tuning parameters", + "text": "Boosted tree tuning parameters\nMost modern boosting methods have a lot of tuning parameters!\n\n\nFor tree growth and pruning (min_n, max_depth, etc)\nFor boosting (trees, stop_iter, learn_rate)\n\n\n\nWe’ll use early stopping to stop boosting when a few iterations produce consecutively worse results." }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#data-splitting-strategy", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#data-splitting-strategy", - "title": "5 - Feature engineering", - "section": "Data splitting strategy", - "text": "Data splitting strategy" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#comparing-tree-ensembles", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#comparing-tree-ensembles", + "title": "6 - Tuning Hyperparameters", + "section": "Comparing tree ensembles", + "text": "Comparing tree ensembles\n\n\nRandom forest\n\nIndependent trees\nBootstrapped data\nNo pruning\n1000’s of trees\n\n\nBoosting\n\nDependent trees\nDifferent case weights\nTune tree parameters\nFar fewer trees\n\n\nThe general consensus for tree-based models is, in terms of performance: boosting > random forest > bagging > single trees." }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#why-a-validation-set", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#why-a-validation-set", - "title": "5 - Feature engineering", - "section": "Why a validation set?", - "text": "Why a validation set?\nRecall that resampling gives us performance measures without using the test set.\nIt’s important to get good resampling statistics (e.g. \\(R^2\\)).\n\nThat usually means having enough data to estimate performance.\n\nWhen you have “a lot” of data, a validation set can be an efficient way to do this." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-tree-code", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-tree-code", + "title": "6 - Tuning Hyperparameters", + "section": "Boosted tree code", + "text": "Boosted tree code\n\nxgb_spec <-\n boost_tree(\n trees = 500, min_n = tune(), stop_iter = tune(), tree_depth = tune(),\n learn_rate = tune(), loss_reduction = tune()\n ) %>%\n set_mode(\"classification\") %>% \n set_engine(\"xgboost\", validation = 1/10) # <- for better early stopping\n\nxgb_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_lencode_mixed(player, outcome = vars(on_goal)) %>% \n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\nxgb_wflow <- \n workflow() %>% \n add_model(xgb_spec) %>% \n add_recipe(xgb_rec)\n\n\nvalidation is an argument to parsnip::xgb_train(), not directly to xgboost. It generates a validation set that is used by xgboost when evaluating model performance. It is eventually assigned to xgb.train(watchlist = list(validation = data)).\nSee translate(xgb_spec) to see where it is passed to parsnip::xgb_train()." }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#splitting-the-nhl-data", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#splitting-the-nhl-data", - "title": "5 - Feature engineering", - "section": "Splitting the NHL data ", - "text": "Splitting the NHL data \n\nset.seed(23)\nnhl_split <- initial_split(season_2015, prop = 3/4)\nnhl_split\n#> <Training/Testing/Total>\n#> <9110/3037/12147>\n\nnhl_train_and_val <- training(nhl_split)\nnhl_test <- testing(nhl_split)\n\n## not testing\nnrow(nhl_train_and_val)\n#> [1] 9110\n \n## testing\nnrow(nhl_test)\n#> [1] 3037" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-3", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-3", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nCreate your boosted tree workflow.\n\n\n\n03:00" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#validation-split", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#validation-split", - "title": "5 - Feature engineering", - "section": "Validation split ", - "text": "Validation split \nSince there are a lot of observations, we’ll use a validation set:\n\nset.seed(234)\nnhl_val <- validation_split(nhl_train_and_val, prop = 0.80)\nnhl_val\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7288/1822]> validation\n\n\nRemember that a validation split is a type of resample." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#running-in-parallel", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#running-in-parallel", + "title": "6 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallel::detectCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn", - "title": "5 - Feature engineering", - "section": "Your turn", - "text": "Your turn\n\nLet’s explore the training set data.\nUse the function plot_nhl_shots() for nice spatial plots of the data.\n\n\n\nnhl_train <- analysis(nhl_val$splits[[1]])\n\nset.seed(100)\nnhl_train %>% \n sample_n(200) %>%\n plot_nhl_shots(emphasis = position)\n\n\n\n\n\n\n\n\n\n\n\n\n08:00" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#running-in-parallel-1", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#running-in-parallel-1", + "title": "6 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#prepare-your-data-for-modeling", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#prepare-your-data-for-modeling", - "title": "5 - Feature engineering", - "section": "Prepare your data for modeling ", - "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of feature engineering steps that provide preprocessing tools to be applied to data.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning ", + "text": "Tuning \nThis will take some time to run ⏳\n\nset.seed(9)\n\nxgb_res <-\n xgb_wflow %>%\n tune_grid(resamples = nhl_val, grid = 15, control = ctrl) # automatic grid now!" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#a-first-recipe", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#a-first-recipe", - "title": "5 - Feature engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-4", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-4", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nStart tuning the boosted tree model!\nWe won’t wait for everyone’s tuning to finish, but take this time to get it started before we move on.\n\n\n\n03:00" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#a-first-recipe-1", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#a-first-recipe-1", - "title": "5 - Feature engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nsummary(nhl_rec)\n#> # A tibble: 17 × 4\n#> variable type role source \n#> <chr> <chr> <chr> <chr> \n#> 1 period numeric predictor original\n#> 2 period_type nominal predictor original\n#> 3 coord_x numeric predictor original\n#> 4 coord_y numeric predictor original\n#> 5 game_time numeric predictor original\n#> 6 strength nominal predictor original\n#> 7 player nominal predictor original\n#> 8 player_diff numeric predictor original\n#> 9 offense_team nominal predictor original\n#> 10 defense_team nominal predictor original\n#> 11 offense_goal_diff numeric predictor original\n#> 12 game_type nominal predictor original\n#> 13 position nominal predictor original\n#> 14 dow nominal predictor original\n#> 15 month nominal predictor original\n#> 16 year numeric predictor original\n#> 17 on_goal nominal outcome original" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-2", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-2", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\nxgb_res\n#> # Tuning results\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7288/1822]> validation <tibble [30 × 9]> <tibble [0 × 3]> <tibble [27,330 × 11]>" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#create-indicator-variables", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#create-indicator-variables", - "title": "5 - Feature engineering", - "section": "Create indicator variables ", - "text": "Create indicator variables \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-3", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-3", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\nautoplot(xgb_res)" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#filter-out-constant-columns", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#filter-out-constant-columns", - "title": "5 - Feature engineering", - "section": "Filter out constant columns ", - "text": "Filter out constant columns \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#again-with-the-location-features", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#again-with-the-location-features", + "title": "6 - Tuning Hyperparameters", + "section": "Again with the location features", + "text": "Again with the location features\n\ncoord_rec <- \n xgb_rec %>%\n step_mutate(\n angle = abs(atan2(abs(coord_y), (89 - abs(coord_x))) * (180 / pi)),\n distance = sqrt((89 - abs(coord_x))^2 + abs(coord_y)^2),\n distance = log(distance),\n behind_goal_line = ifelse(abs(coord_x) >= 89, 1, 0)\n ) %>% \n step_rm(coord_x, coord_y)\n\nxgb_coord_wflow <- \n workflow() %>% \n add_model(xgb_spec) %>% \n add_recipe(coord_rec)\n\nset.seed(9)\nxgb_coord_res <-\n xgb_coord_wflow %>%\n tune_grid(resamples = nhl_val, grid = 20, control = ctrl)" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#normalization", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#normalization", - "title": "5 - Feature engineering", - "section": "Normalization ", - "text": "Normalization \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#did-the-machine-figure-it-out", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#did-the-machine-figure-it-out", + "title": "6 - Tuning Hyperparameters", + "section": "Did the machine figure it out?", + "text": "Did the machine figure it out?\n\nshow_best(xgb_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 11\n#> min_n tree_depth learn_rate loss_reduction stop_iter .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <dbl> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 19 2 0.311 1.46e- 4 14 roc_auc binary 0.633 1 NA Preprocessor1_Model12\n#> 2 11 3 0.0255 5.61e- 7 19 roc_auc binary 0.628 1 NA Preprocessor1_Model05\n#> 3 27 5 0.0120 6.04e- 6 12 roc_auc binary 0.627 1 NA Preprocessor1_Model07\n#> 4 25 14 0.0379 3.62e- 5 8 roc_auc binary 0.626 1 NA Preprocessor1_Model13\n#> 5 31 11 0.00585 1.02e-10 7 roc_auc binary 0.625 1 NA Preprocessor1_Model08\n\nshow_best(xgb_coord_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 11\n#> min_n tree_depth learn_rate loss_reduction stop_iter .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <dbl> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 30 13 0.0578 0.00000000141 18 roc_auc binary 0.648 1 NA Preprocessor1_Model07\n#> 2 39 12 0.0803 0.000411 10 roc_auc binary 0.643 1 NA Preprocessor1_Model12\n#> 3 14 2 0.146 0.00244 19 roc_auc binary 0.642 1 NA Preprocessor1_Model14\n#> 4 26 15 0.0365 2.51 17 roc_auc binary 0.642 1 NA Preprocessor1_Model11\n#> 5 35 5 0.101 0.0000000784 13 roc_auc binary 0.641 1 NA Preprocessor1_Model17" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#reduce-correlation", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#reduce-correlation", - "title": "5 - Feature engineering", - "section": "Reduce correlation ", - "text": "Reduce correlation \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#compare-models", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#compare-models", + "title": "6 - Tuning Hyperparameters", + "section": "Compare models", + "text": "Compare models\nBest logistic regression results:\n\nglm_spline_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, .estimator, mean, n, std_err, .config)\n#> # A tibble: 1 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 roc_auc binary 0.653 1 NA Preprocessor12_Model1\n\n\nBest boosting results:\n\nxgb_coord_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, .estimator, mean, n, std_err, .config)\n#> # A tibble: 1 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 roc_auc binary 0.648 1 NA Preprocessor1_Model07" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps", - "title": "5 - Feature engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-5", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-5", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nCan you get better ROC results with xgboost?\nTry increasing learn_rate beyond the original range.\n\n\n\n20:00" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps-1", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps-1", - "title": "5 - Feature engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = on_goal)\n\n\nA fancy machine learning supervised dimension reduction technique…\n\nNote that this uses the outcome, and it is from an extension package" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#updating-the-workflow", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#updating-the-workflow", + "title": "6 - Tuning Hyperparameters", + "section": "Updating the workflow ", + "text": "Updating the workflow \n\nbest_auc <- select_best(glm_spline_res, metric = \"roc_auc\")\nbest_auc\n#> # A tibble: 1 × 3\n#> angle distance .config \n#> <int> <int> <chr> \n#> 1 5 8 Preprocessor12_Model1\n\nglm_spline_wflow <-\n glm_spline_wflow %>% \n finalize_workflow(best_auc)\n\nglm_spline_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: logistic_reg()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 8 Recipe Steps\n#> \n#> • step_lencode_mixed()\n#> • step_dummy()\n#> • step_mutate()\n#> • step_rm()\n#> • step_zv()\n#> • step_ns()\n#> • step_ns()\n#> • step_normalize()\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps-2", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps-2", - "title": "5 - Feature engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_ns(coord_y, coord_x, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#the-final-fit-to-the-nhl-data", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#the-final-fit-to-the-nhl-data", + "title": "6 - Tuning Hyperparameters", + "section": "The final fit to the NHL data ", + "text": "The final fit to the NHL data \n\ntest_res <- \n glm_spline_wflow %>% \n last_fit(split = nhl_split)\n\ntest_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [9110/3037]> train/test split <tibble [2 × 4]> <tibble [1 × 3]> <tibble [3,037 × 6]> <workflow>\n#> \n#> There were issues with some computations:\n#> \n#> - Warning(s) x1: prediction from a rank-deficient fit may be misleading\n#> \n#> Run `show_notes(.Last.tune.result)` for more information.\n\n\nRemember that last_fit() fits one time with the combined training and validation set, then evaluates one time with the testing set." }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-1", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-1", - "title": "5 - Feature engineering", + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-6", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-6", + "title": "6 - Tuning Hyperparameters", "section": "Your turn", - "text": "Your turn\n\nCreate a recipe() for the on-goal data to :\n\ncreate one-hot indicator variables\nremove zero-variance variables\n\n\n\n\n03:00" + "text": "Your turn\n\nFinalize your workflow with the best parameters.\nCreate a final fit.\n\n\n\n08:00" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#minimal-recipe", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#minimal-recipe", - "title": "5 - Feature engineering", - "section": "Minimal recipe ", - "text": "Minimal recipe \n\nnhl_indicators <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#estimates-of-roc-auc", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#estimates-of-roc-auc", + "title": "6 - Tuning Hyperparameters", + "section": "Estimates of ROC AUC ", + "text": "Estimates of ROC AUC \nValidation results from tuning:\n\nglm_spline_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, mean, n, std_err)\n#> # A tibble: 1 × 4\n#> .metric mean n std_err\n#> <chr> <dbl> <int> <dbl>\n#> 1 roc_auc 0.653 1 NA\n\n\nTest set results:\n\ntest_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.616 Preprocessor1_Model1\n#> 2 roc_auc binary 0.656 Preprocessor1_Model1" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#using-a-workflow", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#using-a-workflow", - "title": "5 - Feature engineering", - "section": "Using a workflow ", - "text": "Using a workflow \n\nset.seed(9)\n\nnhl_glm_wflow <-\n workflow() %>%\n add_recipe(nhl_indicators) %>%\n add_model(logistic_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nnhl_glm_res <-\n nhl_glm_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_glm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.555 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.558 1 NA Preprocessor1_Model1" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#final-fitted-workflow", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#final-fitted-workflow", + "title": "6 - Tuning Hyperparameters", + "section": "Final fitted workflow", + "text": "Final fitted workflow\nExtract the final fitted workflow, fit using the training set:\n\nfinal_glm_spline_wflow <- \n test_res %>% \n extract_workflow()\n\n# use this object to predict or deploy\npredict(final_glm_spline_wflow, nhl_test[1:3,])\n#> # A tibble: 3 × 1\n#> .pred_class\n#> <fct> \n#> 1 no \n#> 2 yes \n#> 3 no" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-2", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-2", - "title": "5 - Feature engineering", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n05:00" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#next-steps", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#next-steps", + "title": "6 - Tuning Hyperparameters", + "section": "Next steps", + "text": "Next steps\n\nDocument the model.\n\n\n\nDeploy the model.\n\n\n\n\nCreate an applicability domain model to help monitor our data over time.\n\n\n\n\nUse explainers to characterize the model and the predictions." }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#holdout-predictions", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#holdout-predictions", - "title": "5 - Feature engineering", - "section": "Holdout predictions ", - "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nglm_val_pred <- collect_predictions(nhl_glm_res)\nglm_val_pred %>% slice(1:7)\n#> # A tibble: 7 × 7\n#> id .pred_yes .pred_no .row .pred_class on_goal .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 validation 0.198 8.02e- 1 10 no no Preprocessor1_Model1\n#> 2 validation 0.264 7.36e- 1 17 no yes Preprocessor1_Model1\n#> 3 validation 0.189 8.11e- 1 23 no no Preprocessor1_Model1\n#> 4 validation 1.00 8.39e-11 40 yes yes Preprocessor1_Model1\n#> 5 validation 0.322 6.78e- 1 41 no yes Preprocessor1_Model1\n#> 6 validation 1.00 8.39e-11 46 yes no Preprocessor1_Model1\n#> 7 validation 0.354 6.46e- 1 55 no no Preprocessor1_Model1" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-yourself", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-yourself", + "title": "6 - Tuning Hyperparameters", + "section": "Explain yourself", + "text": "Explain yourself\nThere are two categories of model explanations, global and local.\n\n\nGlobal model explanations provide an overall understanding aggregated over a whole set of observations.\nLocal model explanations provide information about a prediction for a single observation.\n\n\n\nYou can also build global model explanations by aggregating local model explanations." }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#two-class-data-1", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#two-class-data-1", - "title": "5 - Feature engineering", - "section": "Two class data", - "text": "Two class data\nThese definitions assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#a-tidymodels-explainer", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#a-tidymodels-explainer", + "title": "6 - Tuning Hyperparameters", + "section": "A tidymodels explainer", + "text": "A tidymodels explainer\nWe can build explainers using:\n\noriginal, basic predictors\nderived features\n\n\nlibrary(DALEXtra)\n\nglm_explainer <- explain_tidymodels(\n final_glm_spline_wflow,\n data = dplyr::select(nhl_train, -on_goal),\n # DALEX required an integer for factors:\n y = as.integer(nhl_train$on_goal),\n verbose = FALSE\n)" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curves", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curves", - "title": "5 - Feature engineering", - "section": "ROC curves", - "text": "ROC curves\nTo make an ROC (receiver operator characteristic) curve, we:\n\ncalculate the sensitivity and specificity for all possible thresholds\nplot false positive rate (x-axis) versus true positive rate (y-axis)\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates", + "title": "6 - Tuning Hyperparameters", + "section": "Explain the x coordinates", + "text": "Explain the x coordinates\nWith our explainer, let’s create partial dependence profiles:\n\nset.seed(123)\npdp_coord_x <- model_profile(\n glm_explainer,\n variables = \"coord_x\",\n N = 500,\n groups = \"position\"\n)\n\n\nYou can use the default plot() method or create your own visualization." }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curves-1", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curves-1", - "title": "5 - Feature engineering", - "section": "ROC curves ", - "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\nroc_curve_points <- glm_val_pred %>% roc_curve(truth = on_goal, estimate = .pred_yes)\nroc_curve_points %>% slice(1, 50, 100)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.139 0.0303 0.977\n#> 3 0.272 0.0642 0.955\n\nglm_val_pred %>% roc_auc(truth = on_goal, estimate = .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.558" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates-1", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates-1", + "title": "6 - Tuning Hyperparameters", + "section": "Explain the x coordinates", + "text": "Explain the x coordinates" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curve-plot", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curve-plot", - "title": "5 - Feature engineering", - "section": "ROC curve plot ", - "text": "ROC curve plot \n\nautoplot(roc_curve_points)" + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates-2", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates-2", + "title": "6 - Tuning Hyperparameters", + "section": "Explain the x coordinates", + "text": "Explain the x coordinates" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-3", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-3", - "title": "5 - Feature engineering", + "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-7", + "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-7", + "title": "6 - Tuning Hyperparameters", "section": "Your turn", - "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\n\n\n\n05:00" + "text": "Your turn\n\nCreate an explainer for our glm model.\nTry grouping by another variable, like game_type or dow.\n\n\n\n05:00" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-4", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-4", - "title": "5 - Feature engineering", - "section": "Your turn", - "text": "Your turn\n\nWhat data are being used for this ROC curve plot?\n\n\n\n03:00" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 rsq standard 0.380\n#> 3 mae standard 40.2\n\n\n\nRMSE: difference between the predicted and observed values ⬇️\n\\(R^2\\): squared correlation between the predicted and observed values ⬆️\nMAE: similar to RMSE, but mean absolute error ⬇️" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-do-we-do-with-the-player-data", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-do-we-do-with-the-player-data", - "title": "5 - Feature engineering", - "section": "What do we do with the player data? 🏒", - "text": "What do we do with the player data? 🏒\nThere are 597 unique player values in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the player column with the estimated effect of that predictor\n\n\n\nLet’s use an effect encoding." + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-1", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-is-an-effect-encoding", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-is-an-effect-encoding", - "title": "5 - Feature engineering", - "section": "What is an effect encoding?", - "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> on_goal player .row\n#> <fct> <fct> <int>\n#> 1 yes brian_dumoulin 1\n#> 2 yes patric_hornqvist 2\n#> 3 yes nikita_nesterov 3\n#> 4 yes jack_eichel 4\n#> 5 yes justin_williams 5\n#> 6 yes seth_jones 6\n#> 7 yes kris_letang 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> on_goal player .row\n#> <fct> <dbl> <int>\n#> 1 yes -0.114 1\n#> 2 yes 0.631 2\n#> 3 yes 0.142 3\n#> 4 yes 0.220 4\n#> 5 yes 0.248 5\n#> 6 yes 0.0733 6\n#> 7 yes 0.0774 7\n\n\nThe player column is replaced with the log-odds of being on goal.\n\nAs a reminder:\n\\[\\text{log-odds} = log\\left(\\frac{\\hat{p}}{1 - \\hat{p}}\\right)\\]\nwhere \\(\\hat{p}\\) is the on goal rate estimate.\nFor logistic regression, this is what the predictors are modeling. The log-odds are more likely to be linear with the outcome." + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-2", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n group_by(reflex) %>%\n rmse(latency, .pred)\n#> # A tibble: 3 × 4\n#> reflex .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 low rmse standard 94.3\n#> 2 mid rmse standard 101. \n#> 3 full rmse standard 51.2" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#per-player-statistics", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#per-player-statistics", - "title": "5 - Feature engineering", - "section": "Per-player statistics", - "text": "Per-player statistics\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these rates use partial pooling.\nPooling borrows strength across players and shrinks extreme values (e.g. zero or one) towards the mean for players with very few shots.\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for players with fewer shots by shrinking the estimate to the overall on-goal rate (55.2%)" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-3", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-3", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\nfrog_metrics <- metric_set(rmse, msd)\naugment(tree_fit, new_data = frog_test) %>%\n frog_metrics(latency, .pred)\n#> # A tibble: 2 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 msd standard -0.908" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#partial-pooling", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#partial-pooling", - "title": "5 - Feature engineering", - "section": "Partial pooling", - "text": "Partial pooling" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-1", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#player-effects", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#player-effects", - "title": "5 - Feature engineering", - "section": "Player effects ", - "text": "Player effects \n\nlibrary(embed)\n\nnhl_effect_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_lencode_mixed(player, outcome = vars(on_goal)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-2", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#recipes-are-estimated", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#recipes-are-estimated", - "title": "5 - Feature engineering", - "section": "Recipes are estimated ", - "text": "Recipes are estimated \nPreprocessing steps in a recipe use the training set to compute quantities.\n\nWhat kind of quantities are computed for preprocessing?\n\nLevels of a factor\nWhether a column has zero variance\nNormalization\nFeature extraction\nEffect encodings\n\n\n\nWhen a recipe is part of a workflow, this estimation occurs when fit() is called." + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-3", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-3", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\ntree_fit %>%\n augment(frog_train)\n#> # A tibble: 456 × 6\n#> treatment reflex age t_o_d latency .pred\n#> <chr> <fct> <dbl> <fct> <dbl> <dbl>\n#> 1 control full 5.42 morning 33 39.8\n#> 2 control full 5.38 morning 19 66.7\n#> 3 control full 5.38 morning 2 66.7\n#> 4 control full 5.44 morning 39 39.8\n#> 5 control full 5.41 morning 42 39.8\n#> 6 control full 4.75 afternoon 20 59.8\n#> 7 control full 4.95 night 31 83.1\n#> 8 control full 5.42 morning 21 39.8\n#> 9 gentamicin full 5.39 morning 30 64.6\n#> 10 control full 4.55 afternoon 43 174. \n#> # … with 446 more rows\n#> # ℹ Use `print(n = ...)` to see more rows\n\nWe call this “resubstitution” or “repredicting the training set”" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#effect-encoding-results", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#effect-encoding-results", - "title": "5 - Feature engineering", - "section": "Effect encoding results ", - "text": "Effect encoding results \n\nnhl_effect_wflow <-\n nhl_glm_wflow %>%\n update_recipe(nhl_effect_rec)\n\nnhl_effect_res <-\n nhl_effect_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.540 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.551 1 NA Preprocessor1_Model1" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-4", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-4", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4\n\nWe call this a “resubstitution estimate”" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#angle", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#angle", - "title": "5 - Feature engineering", - "section": "Angle", - "text": "Angle\n\nnhl_angle_rec <-\n nhl_indicators %>%\n step_mutate(\n angle = abs(atan2(abs(coord_y), (89 - abs(coord_x))) * (180 / pi))\n )\n\n\n\nNote the danger of using step_mutate() – easy to have data leakage\ncoord_x is “distance from goal”. We subtract it from 89 to get the distance from the center of the ice. The abs() calls account for the fact that the goals might be on either side of (0, 0). The rest of it is the formula for going from (x, y) to angle in degrees." + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-5", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-5", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#distance", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#distance", - "title": "5 - Feature engineering", - "section": "Distance", - "text": "Distance\n\nnhl_distance_rec <-\n nhl_angle_rec %>%\n step_mutate(\n distance = sqrt((89 - abs(coord_x))^2 + abs(coord_y)^2),\n distance = log(distance)\n )" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-6", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-6", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4\n\n\n\ntree_fit %>%\n augment(frog_test) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#behind-goal-line", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#behind-goal-line", - "title": "5 - Feature engineering", - "section": "Behind goal line", - "text": "Behind goal line\n\nnhl_behind_rec <-\n nhl_distance_rec %>%\n step_mutate(\n behind_goal_line = ifelse(abs(coord_x) >= 89, 1, 0)\n )" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse augment() and metrics() to compute a regression metric like mae().\nCompute the metrics for both training and testing data.\nNotice the evidence of overfitting! ⚠️\n\n\n\n05:00" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#fit-different-recipes", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#fit-different-recipes", - "title": "5 - Feature engineering", - "section": "Fit different recipes ", - "text": "Fit different recipes \nA workflow set can cross models and/or preprocessors and then resample them en masse.\n\nset.seed(9)\n\nnhl_glm_set_res <-\n workflow_set(\n list(`1_dummy` = nhl_indicators, `2_angle` = nhl_angle_rec, \n `3_dist` = nhl_distance_rec, `4_bgl` = nhl_behind_rec),\n list(logistic = logistic_reg())\n ) %>%\n workflow_map(fn = \"fit_resamples\", resamples = nhl_val, verbose = TRUE, control = ctrl)" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-7", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-7", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4 \n#> 2 rsq standard 0.494\n#> 3 mae standard 33.4\n\n\n\ntree_fit %>%\n augment(frog_test) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 rsq standard 0.380\n#> 3 mae standard 40.2\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-5", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-5", - "title": "5 - Feature engineering", - "section": "Your turn", - "text": "Your turn\n\nCreate a workflow set with 2 or 3 recipes.\n(Consider using recipes we’ve already created.)\nUse workflow_map() to resample the workflow set.\n\n\n\n08:00" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#compare-recipes", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#compare-recipes", - "title": "5 - Feature engineering", - "section": "Compare recipes", - "text": "Compare recipes\n\nlibrary(forcats)\ncollect_metrics(nhl_glm_set_res) %>%\n filter(.metric == \"roc_auc\") %>%\n mutate(\n features = gsub(\"_logistic\", \"\", wflow_id), \n features = fct_reorder(features, mean)\n ) %>%\n ggplot(aes(x = mean, y = features)) +\n geom_point(size = 3) +\n labs(y = NULL, x = \"ROC AUC (validation set)\")" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-1", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#compare-recipes-1", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#compare-recipes-1", - "title": "5 - Feature engineering", - "section": "Compare recipes", - "text": "Compare recipes" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-1", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n03:00" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#debugging-a-recipe", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#debugging-a-recipe", - "title": "5 - Feature engineering", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. encoded_players) can be estimated manually with a function called prep(). It is analogous to fit().\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back." + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-2", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(frog_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [410/46]> Fold01\n#> 2 <split [410/46]> Fold02\n#> 3 <split [410/46]> Fold03\n#> 4 <split [410/46]> Fold04\n#> 5 <split [410/46]> Fold05\n#> 6 <split [410/46]> Fold06\n#> 7 <split [411/45]> Fold07\n#> 8 <split [411/45]> Fold08\n#> 9 <split [411/45]> Fold09\n#> 10 <split [411/45]> Fold10" }, { - "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#more-on-recipes", - "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#more-on-recipes", - "title": "5 - Feature engineering", - "section": "More on recipes", - "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-3", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-3", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWhat is in this?\n\nfrog_folds <- vfold_cv(frog_train)\nfrog_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n\n\nTalk about a list column, storing non-atomic types in dataframe" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n03:00\n\n\n\n\n\nlm for linear model\nglm for generalized linear model (e.g. logistic regression)\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-4", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-4", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(frog_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [364/92]> Fold1\n#> 2 <split [365/91]> Fold2\n#> 3 <split [365/91]> Fold3\n#> 4 <split [365/91]> Fold4\n#> 5 <split [365/91]> Fold5" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-5", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-5", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(frog_train, strata = latency)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [408/48]> Fold01\n#> 2 <split [408/48]> Fold02\n#> 3 <split [408/48]> Fold03\n#> 4 <split [409/47]> Fold04\n#> 5 <split [411/45]> Fold05\n#> 6 <split [412/44]> Fold06\n#> 7 <split [412/44]> Fold07\n#> 8 <split [412/44]> Fold08\n#> 9 <split [412/44]> Fold09\n#> 10 <split [412/44]> Fold10\n\n\nStratification often helps, with very little downside" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-1", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-1", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlinear_reg()\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: lm\n\n\nModels have default engines" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-6", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-6", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nfrog_folds <- vfold_cv(frog_train, v = 10, strata = latency)\nfrog_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [408/48]> Fold01\n#> 2 <split [408/48]> Fold02\n#> 3 <split [408/48]> Fold03\n#> 4 <split [409/47]> Fold04\n#> 5 <split [411/45]> Fold05\n#> 6 <split [412/44]> Fold06\n#> 7 <split [412/44]> Fold07\n#> 8 <split [412/44]> Fold08\n#> 9 <split [412/44]> Fold09\n#> 10 <split [412/44]> Fold10\n\n\nSet the seed when creating resamples" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-2", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-2", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#fit-our-model-to-the-resamples", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#fit-our-model-to-the-resamples", + "title": "4 - Evaluating models", + "section": "Fit our model to the resamples", + "text": "Fit our model to the resamples\n\ntree_res <- fit_resamples(tree_wflow, frog_folds)\ntree_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]>" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-3", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-3", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlinear_reg() %>%\n set_engine(\"glmnet\")\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: glmnet" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\ntree_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 rmse standard 59.6 10 2.31 Preprocessor1_Model1\n#> 2 rsq standard 0.305 10 0.0342 Preprocessor1_Model1\n\n\nWe can reliably measure performance using only the training data 🎉" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-4", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-4", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlinear_reg() %>%\n set_engine(\"stan\")\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: stan" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#comparing-metrics", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#comparing-metrics", + "title": "4 - Evaluating models", + "section": "Comparing metrics ", + "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\ntree_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 rmse 59.6 10\n#> 2 rsq 0.305 10\n\n\nThe RMSE previously was\n\n49.36 for the training set\n59.16 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-5", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-5", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance-1", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_frog <- control_resamples(save_pred = TRUE)\ntree_res <- fit_resamples(tree_wflow, frog_folds, control = ctrl_frog)\n\ntree_preds <- collect_predictions(tree_res)\ntree_preds\n#> # A tibble: 456 × 5\n#> id .pred .row latency .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 Fold01 39.6 1 33 Preprocessor1_Model1\n#> 2 Fold01 72.1 3 2 Preprocessor1_Model1\n#> 3 Fold01 63.8 9 30 Preprocessor1_Model1\n#> 4 Fold01 72.1 13 46 Preprocessor1_Model1\n#> 5 Fold01 43.3 28 11 Preprocessor1_Model1\n#> 6 Fold01 61.7 35 41 Preprocessor1_Model1\n#> 7 Fold01 39.6 51 43 Preprocessor1_Model1\n#> 8 Fold01 134. 70 20 Preprocessor1_Model1\n#> 9 Fold01 70.6 74 21 Preprocessor1_Model1\n#> 10 Fold01 39.6 106 14 Preprocessor1_Model1\n#> # … with 446 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-6", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-6", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-3", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-3", + "title": "4 - Evaluating models", + "section": "", + "text": "tree_preds %>% \n ggplot(aes(latency, .pred, color = id)) + \n geom_abline(lty = 2, col = \"gray\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-7", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-7", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"regression\")\n#> Decision Tree Model Specification (regression)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#where-are-the-fitted-models", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#where-are-the-fitted-models", + "title": "4 - Evaluating models", + "section": "Where are the fitted models? ", + "text": "Where are the fitted models? \n\ntree_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [47 × 4]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [45 × 4]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n\n\n🗑️" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-8", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-8", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping", + "title": "4 - Evaluating models", + "section": "Bootstrapping", + "text": "Bootstrapping" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-1", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-1", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code so it creates a different model.\n\n\n\n05:00\n\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping-1", + "title": "4 - Evaluating models", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(frog_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [456/163]> Bootstrap01\n#> 2 <split [456/166]> Bootstrap02\n#> 3 <split [456/173]> Bootstrap03\n#> 4 <split [456/177]> Bootstrap04\n#> 5 <split [456/166]> Bootstrap05\n#> 6 <split [456/163]> Bootstrap06\n#> 7 <split [456/164]> Bootstrap07\n#> 8 <split [456/165]> Bootstrap08\n#> 9 <split [456/170]> Bootstrap09\n#> 10 <split [456/177]> Bootstrap10\n#> # … with 15 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#models-well-be-using-today", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#models-well-be-using-today", - "title": "3 - What makes a model?", - "section": "Models we’ll be using today", - "text": "Models we’ll be using today\n\nLinear regression\nDecision trees" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-2", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCreate:\n\nbootstrap folds (change times from the default)\nvalidation set (use the reference guide to find the function)\n\nDon’t forget to set a seed when you resample!\n\n\n\n05:00" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression", - "title": "3 - What makes a model?", - "section": "Linear regression", - "text": "Linear regression" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping-2", + "title": "4 - Evaluating models", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(322)\nbootstraps(frog_train, times = 10)\n#> # Bootstrap sampling \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [456/173]> Bootstrap01\n#> 2 <split [456/168]> Bootstrap02\n#> 3 <split [456/170]> Bootstrap03\n#> 4 <split [456/164]> Bootstrap04\n#> 5 <split [456/176]> Bootstrap05\n#> 6 <split [456/156]> Bootstrap06\n#> 7 <split [456/166]> Bootstrap07\n#> 8 <split [456/168]> Bootstrap08\n#> 9 <split [456/167]> Bootstrap09\n#> 10 <split [456/170]> Bootstrap10" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression-1", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression-1", - "title": "3 - What makes a model?", - "section": "Linear regression", - "text": "Linear regression" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#validation-set", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#validation-set", + "title": "4 - Evaluating models", + "section": "Validation set ", + "text": "Validation set \n\nset.seed(853)\nvalidation_split(frog_train, strata = latency)\n#> # Validation Set Split (0.75/0.25) using stratification \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [340/116]> validation\n\n\nA validation set is just another type of resample" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression-2", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression-2", - "title": "3 - What makes a model?", - "section": "Linear regression", - "text": "Linear regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nOutcome modeled as linear combination of predictors:\n\n\\(\\mbox{latency} = \\beta_0 + \\beta_1\\cdot\\mbox{age} + \\epsilon\\)\n\nFind a line that minimizes the mean squared error (MSE)" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#random-forest-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#random-forest-1", + "title": "4 - Evaluating models", + "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this tomorrow!), as long as there are enough trees" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#create-a-random-forest-model", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#create-a-random-forest-model", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"regression\")\nrf_spec\n#> Random Forest Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees-1", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees-1", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#create-a-random-forest-model-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#create-a-random-forest-model-1", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_wflow <- workflow(latency ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees-2", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees-2", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-3", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-3", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\nplot true vs. predicted values\n\n\n\n\n08:00" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "title": "3 - What makes a model?", - "section": "All models are wrong, but some are useful!", - "text": "All models are wrong, but some are useful!\n\n\nLinear regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance-2", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nctrl_frog <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, frog_folds, control = ctrl_frog)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 rmse standard 55.9 10 1.71 Preprocessor1_Model1\n#> 2 rsq standard 0.370 10 0.0306 Preprocessor1_Model1" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "title": "3 - What makes a model?", - "section": "Workflows bind preprocessors and models", - "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-5", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-5", + "title": "4 - Evaluating models", + "section": "", + "text": "collect_predictions(rf_res) %>% \n ggplot(aes(latency, .pred, color = id)) + \n geom_abline(lty = 2, col = \"gray\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#what-is-wrong-with-this", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#what-is-wrong-with-this", - "title": "3 - What makes a model?", - "section": "What is wrong with this?", - "text": "What is wrong with this?" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", + "title": "4 - Evaluating models", + "section": "How can we compare multiple model workflows at once?", + "text": "How can we compare multiple model workflows at once?" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#why-a-workflow", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#why-a-workflow", - "title": "3 - What makes a model?", - "section": "Why a workflow()? ", - "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering tomorrow!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set", + "title": "4 - Evaluating models", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec))\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-1", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-1", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\ntree_spec %>% \n fit(latency ~ ., data = frog_train) \n#> parsnip model object\n#> \n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set-1", + "title": "4 - Evaluating models", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = frog_folds)\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-2", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-2", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\nworkflow() %>%\n add_formula(latency ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = frog_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set-2", + "title": "4 - Evaluating models", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = frog_folds) %>%\n rank_results()\n#> # A tibble: 4 × 9\n#> wflow_id .config .metric mean std_err n prepr…¹ model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_forest Prepro… rmse 55.8 1.71 10 formula rand… 1\n#> 2 formula_rand_forest Prepro… rsq 0.371 0.0301 10 formula rand… 1\n#> 3 formula_decision_tree Prepro… rmse 59.6 2.31 10 formula deci… 2\n#> 4 formula_decision_tree Prepro… rsq 0.305 0.0342 10 formula deci… 2\n#> # … with abbreviated variable name ¹​preprocessor\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-3", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-3", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\nworkflow(latency ~ ., tree_spec) %>% \n fit(data = frog_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-4", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-4", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nWhen do you think a workflow set would be useful?\n\n\n\n03:00" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-2", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-2", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code so it uses a linear model.\n\n\n\n05:00" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#the-final-fit", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#the-final-fit", + "title": "4 - Evaluating models", + "section": "The final fit ", + "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# frog_split has train + test info\nfinal_fit <- last_fit(rf_wflow, frog_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [456/116]> train/test split <tibble> <tibble> <tibble> <workflow>" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#predict-with-your-model", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#predict-with-your-model", - "title": "3 - What makes a model?", - "section": "Predict with your model ", - "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\ntree_fit <-\n workflow(latency ~ ., tree_spec) %>% \n fit(data = frog_train)" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 rmse standard 57.1 Preprocessor1_Model1\n#> 2 rsq standard 0.420 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-3", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-3", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = frog_test)\nWhat do you get?\n\n\n\n03:00" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit-1", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 116 × 5\n#> id .pred .row latency .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 train/test split 43.5 1 22 Preprocessor1_Model1\n#> 2 train/test split 104. 3 106 Preprocessor1_Model1\n#> 3 train/test split 76.2 6 39 Preprocessor1_Model1\n#> 4 train/test split 42.4 8 50 Preprocessor1_Model1\n#> 5 train/test split 43.5 10 63 Preprocessor1_Model1\n#> 6 train/test split 43.1 14 25 Preprocessor1_Model1\n#> 7 train/test split 51.5 16 48 Preprocessor1_Model1\n#> 8 train/test split 160. 17 91 Preprocessor1_Model1\n#> 9 train/test split 50.9 32 11 Preprocessor1_Model1\n#> 10 train/test split 171. 33 109 Preprocessor1_Model1\n#> # … with 106 more rows\n#> # ℹ Use `print(n = ...)` to see more rows\n\n\nThese are predictions for the test set" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-4", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-4", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = frog_test)\nWhat do you get?\n\n\n\n03:00" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-6", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-6", + "title": "4 - Evaluating models", + "section": "", + "text": "collect_predictions(final_fit) %>%\n ggplot(aes(latency, .pred)) + \n geom_abline(lty = 2, col = \"deeppink4\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit-2", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1)) \n#> \n#> Type: Regression \n#> Number of trees: 1000 \n#> Sample size: 456 \n#> Number of independent variables: 4 \n#> Mtry: 2 \n#> Target node size: 5 \n#> Variable importance mode: none \n#> Splitrule: variance \n#> OOB prediction error (MSE): 3124.583 \n#> R squared (OOB): 0.3531813\n\n\nUse this for prediction on new data, like for deploying" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model-1", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model-1", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\nroundint = FALSE is only to quiet a warning" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-5", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-5", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nEnd of the day discussion!\nWhich model do you think you would decide to use?\nWhat surprised you the most?\nWhat is one thing you are looking forward to for tomorrow?\n\n\n\n05:00" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model-2", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model-2", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit", + "title": "4 - Evaluating models", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-5", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-5", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nExtract the model engine object from your fitted linear workflow.\n⚠️ Never predict() with any extracted components!\n\n\n\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-1", + "title": "4 - Evaluating models", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#deploying-a-model", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#deploying-a-model", - "title": "3 - What makes a model?", - "section": "Deploying a model ", - "text": "Deploying a model \nHow do you use your new tree_fit model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"frog_hatching\")\nv\n#> \n#> ── frog_hatching ─ <butchered_workflow> model for deployment \n#> A rpart regression modeling workflow using 4 features\n\nLearn more at https://vetiver.rstudio.com" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-2", + "title": "4 - Evaluating models", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#deploy-your-model-1", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#deploy-your-model-1", - "title": "3 - What makes a model?", - "section": "Deploy your model ", - "text": "Deploy your model \nHow do you use your new model tree_fit in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 2 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/vetiver\n#> ├──/ping (GET)\n#> └──/predict (POST)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-3", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-3", + "title": "4 - Evaluating models", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-6", - "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-6", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n05:00" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-4", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-4", + "title": "4 - Evaluating models", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#workshop-policies", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#workshop-policies", - "title": "1 - Introduction", - "section": "Workshop policies", - "text": "Workshop policies\n\nPlease do not photograph people wearing red lanyards\nThere are gender neutral bathrooms near National Harbor rooms\nA meditation room is located at National Harbor 9 (8am - 5pm, Mon - Thurs)\nA lactation room is located at Potomac Dressing Room (8am - 5pm, Mon - Thurs)" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-5", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-5", + "title": "4 - Evaluating models", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#workshop-policies-1", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#workshop-policies-1", - "title": "1 - Introduction", - "section": "Workshop policies", - "text": "Workshop policies\n\nPlease review the rstudio::conf code of conduct, which applies to all workshops: https://www.rstudio.com/conference/2022/2022-conf-code-of-conduct/\nCoC site has info on how to report a problem (in person, email, phone)\nYou are required to wear a mask that fully covers your mouth and nose at all times in all public spaces" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nIteratively add candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-you", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-1", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-1", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nstack_ctrl <- control_resamples(save_pred = TRUE, save_workflow = TRUE)" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-we", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-we", - "title": "1 - Introduction", - "section": "Who are we?", - "text": "Who are we?\n\n\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\n\nJulia Silge\nDavid Robinson\nDavis Vaughan" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-2", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-2", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nDefine candidate members\n\nStart out with a linear regression:\n\nlr_res <- \n # define model spec\n linear_reg() %>%\n set_mode(\"regression\") %>%\n # add to workflow\n workflow(preprocessor = latency ~ .) %>%\n # fit to resamples\n fit_resamples(frog_folds, control = stack_ctrl)" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-we-1", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-we-1", - "title": "1 - Introduction", - "section": "Who are we?", - "text": "Who are we?\n\n\n\nKelly Bodwin\nMichael Chow\nPritam Dalal\nMatt Dancho\nJon Harmon\n\n\n\nMike Mahoney\nEdgar Ruiz\nAsmae Toumi\nQiushi Yan\n\n\n\nMany thanks to Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-3", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-3", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nlr_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [47 × 4]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [45 × 4]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#asking-for-help", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#asking-for-help", - "title": "1 - Introduction", - "section": "Asking for help", - "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-4", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-4", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \nThen, a random forest:\n\nrf_res <- \n # define model spec\n rand_forest() %>%\n set_mode(\"regression\") %>%\n # add to workflow\n workflow(preprocessor = latency ~ .) %>%\n # fit to resamples\n fit_resamples(frog_folds, control = stack_ctrl)" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#section-2", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#section-2", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-5", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-5", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nrf_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [47 × 4]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [45 × 4]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#plan-for-this-workshop", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Plan for this workshop", - "text": "Plan for this workshop\n\nToday:\n\nYour data budget\nWhat makes a model\nEvaluating models\n\nTomorrow:\n\nFeature engineering\nTuning hyperparameters\nWrapping up!" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-6", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-6", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nInitialize a data stack object\n\n\n\nfrog_st <- stacks()\n\nfrog_st\n#> # A data stack with 0 model definitions and 0 candidate members." }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#section-3", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#section-3", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋\n\n Log in to RStudio Cloud here (free):\nbit.ly/tidymodels-rstudioconf-2022" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-7", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-7", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nIteratively add candidate ensemble members to the data stack\n\n\nfrog_st <- frog_st %>%\n add_candidates(lr_res) %>%\n add_candidates(rf_res)\n\nfrog_st\n#> # A data stack with 2 model definitions and 2 candidate members:\n#> # lr_res: 1 model configuration\n#> # rf_res: 1 model configuration\n#> # Outcome: latency (numeric)\n\nTomorrow we’ll discuss tuning parameters where there are different configurations of models (e.g. 10 different variations of the random forest model).\nThese configurations can greatly improve the performance of the stacking ensemble." }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-8", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-8", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nEvaluate how to combine their predictions\n\n\nfrog_st_res <- frog_st %>%\n blend_predictions()\n\nfrog_st_res\n#> # A tibble: 2 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 rf_res_1_1 rand_forest 0.635\n#> 2 lr_res_1_1 linear_reg 0.344" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning-1", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning-1", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-9", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-9", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nFit candidate ensemble members with non-zero stacking coefficients\n\n\nfrog_st_res <- frog_st_res %>%\n fit_members()\n\nfrog_st_res\n#> # A tibble: 2 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 rf_res_1_1 rand_forest 0.635\n#> 2 lr_res_1_1 linear_reg 0.344" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning-2", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning-2", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-10", + "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-10", + "title": "4 - Evaluating models", + "section": "Building a model stack ", + "text": "Building a model stack \n\nPredict on new data!\n\n\n\nfrog_test %>%\n select(latency) %>%\n bind_cols(\n predict(frog_st_res, frog_test)\n ) %>%\n ggplot(aes(latency, .pred)) + \n geom_abline(lty = 2, \n col = \"deeppink4\", \n size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#your-turn", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#your-turn", - "title": "1 - Introduction", - "section": "Your turn", - "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching", + "title": "2 - Your data budget", + "section": "Data on tree frog hatching", + "text": "Data on tree frog hatching" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-tidymodels", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-tidymodels", - "title": "1 - Introduction", - "section": "What is tidymodels? ", - "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.0.0 ──\n#> ✔ broom 1.0.0 ✔ rsample 1.0.0\n#> ✔ dials 1.0.0 ✔ tibble 3.1.8\n#> ✔ dplyr 1.0.9 ✔ tidyr 1.2.0\n#> ✔ infer 1.0.2 ✔ tune 1.0.0\n#> ✔ modeldata 1.0.0 ✔ workflows 1.0.0\n#> ✔ parsnip 1.0.0 ✔ workflowsets 1.0.0\n#> ✔ purrr 0.3.4 ✔ yardstick 1.0.0\n#> ✔ recipes 1.0.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Learn how to get started at https://www.tidymodels.org/start/" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-1", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-1", + "title": "2 - Your data budget", + "section": "Data on tree frog hatching", + "text": "Data on tree frog hatching\n\nRed-eyed tree frog embryos can hatch earlier than their normal ~7 days if they detect potential predator threat!\nType ?stacks::tree_frogs to learn more about this dataset, including references.\nWe are using a slightly modified version from stacks.\n\n\nlibrary(tidymodels)\n\ndata(\"tree_frogs\", package = \"stacks\")\ntree_frogs <- tree_frogs %>%\n mutate(t_o_d = factor(t_o_d),\n age = age / 86400) %>%\n filter(!is.na(latency)) %>%\n select(-c(clutch, hatched))" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#the-whole-game", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#the-whole-game", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\nTomorrow we will walk through a case study in detail to illustrate feature engineering and model tuning.\nToday we will walk through the analysis at a higher level to show the model development process as a whole and give you an introduction to the data set.\nThe data are from the NHL where we want to predict whether a shot was on-goal or not! 🏒\nIt’s a good example to show how model development works." + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-2", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-2", + "title": "2 - Your data budget", + "section": "Data on tree frog hatching", + "text": "Data on tree frog hatching\n\n\n\nN = 572\nA numeric outcome, latency\n4 other variables\n\ntreatment, reflex, and t_o_d are nominal predictors\nage is a numeric predictor\n\n\n\n\n\n\n\nlatency: How long it took the frog to hatch after being stimulated - i.e. after being poked by a blunt probe (in seconds).\ntreatment: Whether or not they got gentamicin, a compound that knocks out the embryo’s lateral line (a sensory organ).\nreflex: A measure of ear function (low, mid, full)\nt_o_d: Time that the stimulus was applied (morning, afternoon, night)\nage: Age at the time it was stimulated (in days)" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#shots-on-goal", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#shots-on-goal", - "title": "1 - Introduction", - "section": "Shots on goal", - "text": "Shots on goal" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-3", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-3", + "title": "2 - Your data budget", + "section": "Data on tree frog hatching", + "text": "Data on tree frog hatching\n\ntree_frogs\n#> # A tibble: 572 × 5\n#> treatment reflex age t_o_d latency\n#> <chr> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.40 morning 22\n#> 2 control low 4.18 night 360\n#> 3 control full 4.65 afternoon 106\n#> 4 control mid 4.14 night 180\n#> 5 control full 4.6 afternoon 60\n#> 6 gentamicin full 5.36 morning 39\n#> 7 control full 4.56 afternoon 214\n#> 8 control full 5.43 morning 50\n#> 9 control full 4.63 afternoon 224\n#> 10 control full 5.40 morning 63\n#> # … with 562 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#data-spending", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#data-spending", - "title": "1 - Introduction", - "section": "Data spending", - "text": "Data spending" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#a-first-model", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#a-first-model", - "title": "1 - Introduction", - "section": "A first model", - "text": "A first model" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-1", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-1", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#starting-point-logistic-regression", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#starting-point-logistic-regression", - "title": "1 - Introduction", - "section": "Starting point: logistic regression", - "text": "Starting point: logistic regression\n\nWe’ll start by using basic logistic regression to predict our binary outcome.\nOur first model will have 16 simple predictor columns.\nOne initial question: there are 640 players taking shots.\nFor logistic regression, do we convert these to binary indicators (a.k.a. “dummies”)?" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-2", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-2", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#basic-features-inc-dummy-variables", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#basic-features-inc-dummy-variables", - "title": "1 - Introduction", - "section": "Basic features (inc dummy variables)", - "text": "Basic features (inc dummy variables)" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n03:00" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#different-player-encoding", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#different-player-encoding", - "title": "1 - Introduction", - "section": "Different player encoding", - "text": "Different player encoding" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-3", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-3", + "title": "2 - Your data budget", + "section": "Data splitting and spending ", + "text": "Data splitting and spending \n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs)\nfrog_split\n#> <Training/Testing/Total>\n#> <429/143/572>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-about-location", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-about-location", - "title": "1 - Introduction", - "section": "What about location", - "text": "What about location\nThe previous models used the x/y coordinates.\nAre there better ways to represent shot location?\nHow can we make location more usable for the model?" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#accessing-the-data", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#accessing-the-data", + "title": "2 - Your data budget", + "section": "Accessing the data ", + "text": "Accessing the data \n\nfrog_train <- training(frog_split)\nfrog_test <- testing(frog_split)" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-angle", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-angle", - "title": "1 - Introduction", - "section": "Add shot angle?", - "text": "Add shot angle?" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#the-training-set", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#the-training-set", + "title": "2 - Your data budget", + "section": "The training set", + "text": "The training set\n\nfrog_train\n#> # A tibble: 429 × 5\n#> treatment reflex age t_o_d latency\n#> <chr> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.36 morning 36\n#> 2 gentamicin full 5.37 morning 72\n#> 3 gentamicin full 4.65 afternoon 141\n#> 4 control full 5.42 morning 27\n#> 5 control full 5.43 morning 27\n#> 6 gentamicin full 5.38 morning 73\n#> 7 gentamicin full 5.42 morning 68\n#> 8 gentamicin full 4.75 afternoon 124\n#> 9 control full 5.00 night 62\n#> 10 control full 5.39 morning 25\n#> # … with 419 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-distance", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-distance", - "title": "1 - Introduction", - "section": "Add shot distance?", - "text": "Add shot distance?" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#the-test-set", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#the-test-set", + "title": "2 - Your data budget", + "section": "The test set ", + "text": "The test set \n\nfrog_test\n#> # A tibble: 143 × 5\n#> treatment reflex age t_o_d latency\n#> <chr> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.40 morning 22\n#> 2 control low 4.18 night 360\n#> 3 control full 4.63 afternoon 224\n#> 4 gentamicin full 4.75 afternoon 158\n#> 5 control mid 4.22 night 91\n#> 6 gentamicin full 4.89 night 301\n#> 7 control full 5.38 morning 2\n#> 8 control full 4.80 afternoon 56\n#> 9 control full 5.36 morning 11\n#> 10 control full 5.40 morning 64\n#> # … with 133 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-behind-goal-line", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-behind-goal-line", - "title": "1 - Introduction", - "section": "Add shot behind goal line?", - "text": "Add shot behind goal line?" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn-1", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn-1", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n05:00" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#nonlinear-terms-for-angle-and-distance", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#nonlinear-terms-for-angle-and-distance", - "title": "1 - Introduction", - "section": "Nonlinear terms for angle and distance", - "text": "Nonlinear terms for angle and distance" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-4", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-4", + "title": "2 - Your data budget", + "section": "Data splitting and spending ", + "text": "Data splitting and spending \n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs, prop = 0.8)\nfrog_train <- training(frog_split)\nfrog_test <- testing(frog_split)\n\nnrow(frog_train)\n#> [1] 457\nnrow(frog_test)\n#> [1] 115" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#try-another-model", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#try-another-model", - "title": "1 - Introduction", - "section": "Try another model", - "text": "Try another model" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-1", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-1", + "title": "2 - Your data budget", + "section": "", + "text": "We will use this tomorrow" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#switch-to-boosting-and-basic-features", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#switch-to-boosting-and-basic-features", - "title": "1 - Introduction", - "section": "Switch to boosting and basic features", - "text": "Switch to boosting and basic features" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn-2", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn-2", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nExplore the frog_train data on your own!\n\nWhat’s the distribution of the outcome, latency?\nWhat’s the distribution of numeric variables like age?\nHow does latency differ across the categorical variables?\n\n\n\n\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#boosting-with-location-features", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#boosting-with-location-features", - "title": "1 - Introduction", - "section": "Boosting with location features", - "text": "Boosting with location features" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-3", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-3", + "title": "2 - Your data budget", + "section": "", + "text": "ggplot(frog_train, aes(latency)) +\n geom_histogram(bins = 20)\n\n\n\nThis histogram brings up a concern. What if in our training set we get unlucky and sample few or none of these large values? That could mean that our model wouldn’t be able to predict such values. Let’s come back to that!" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#choose-wisely", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#choose-wisely", - "title": "1 - Introduction", - "section": "Choose wisely…", - "text": "Choose wisely…" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-4", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-4", + "title": "2 - Your data budget", + "section": "", + "text": "ggplot(frog_train, aes(latency, treatment, fill = treatment)) +\n geom_boxplot(alpha = 0.5, show.legend = FALSE)" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#finalize-and-verify", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#finalize-and-verify", - "title": "1 - Introduction", - "section": "Finalize and verify", - "text": "Finalize and verify" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-5", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-5", + "title": "2 - Your data budget", + "section": "", + "text": "frog_train %>%\n ggplot(aes(latency, reflex, fill = reflex)) +\n geom_boxplot(alpha = 0.3, show.legend = FALSE)" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#and-so-on", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#and-so-on", - "title": "1 - Introduction", - "section": "… and so on", - "text": "… and so on\nOnce we find an acceptable model and feature set, the process is to\n\nConfirm our results on the test set.\nDocument the data and model development process.\nDeploy, monitor, etc." + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-6", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-6", + "title": "2 - Your data budget", + "section": "", + "text": "ggplot(frog_train, aes(age, latency, color = reflex)) +\n geom_point(alpha = .8, size = 2)" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#lets-install-some-packages", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of RStudio Cloud:\n\ninstall.packages(c(\"DALEXtra\", \"doParallel\", \"embed\", \"forcats\",\n \"lme4\", \"ranger\", \"remotes\", \"rpart\", \n \"rpart.plot\", \"stacks\", \"tidymodels\",\n \"vetiver\", \"xgboost\"))\n\nremotes::install_github(\"topepo/ongoal@v0.0.2\")\n\n\n\n Or log in to RStudio Cloud:\nbit.ly/tidymodels-rstudioconf-2022" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-7", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-7", + "title": "2 - Your data budget", + "section": "", + "text": "Stratified sampling would split within each quartile\n\nBased on our exploration, we realized that stratifying by latency might help get a consistent distribution. For instance, we’d include high and low latency in both the test and training" }, { - "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#our-versions", - "href": "archive/2022-07-RStudio-conf/01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nbroom (1.0.0, CRAN), DALEX (2.4.0, CRAN), DALEXtra (2.2.0, CRAN), dials (1.0.0, CRAN), doParallel (1.0.17, CRAN), dplyr (1.0.9, CRAN), embed (1.0.0, CRAN), ggplot2 (3.3.6, CRAN), modeldata (1.0.0, CRAN), ongoal (0.0.2, Github (topepo/ongoal@02cd6b233), parsnip (1.0.0, CRAN), purrr (0.3.4, CRAN), ranger (0.13.1, CRAN), recipes (1.0.1, local), rpart (4.1.16, CRAN), rpart.plot (3.1.1, CRAN), rsample (1.0.0, CRAN), scales (1.2.0, CRAN), stacks (0.2.3, CRAN), tibble (3.1.8, CRAN), tidymodels (1.0.0, CRAN), tidyr (1.2.0, CRAN), tune (1.0.0, CRAN), vetiver (0.1.5, CRAN), workflows (1.0.0, CRAN), workflowsets (1.0.0, CRAN), xgboost (1.6.0.1, CRAN), and yardstick (1.0.0, CRAN)" + "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#stratification", + "href": "archive/2022-07-RStudio-conf/02-data-budget.html#stratification", + "title": "2 - Your data budget", + "section": "Stratification", + "text": "Stratification\nUse strata = latency\n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs, prop = 0.8, strata = latency)\nfrog_split\n#> <Training/Testing/Total>\n#> <456/116/572>\n\n\nStratification often helps, with very little downside" }, { - "objectID": "slides/intro-extra-recipes.html#looking-at-the-predictors", - "href": "slides/intro-extra-recipes.html#looking-at-the-predictors", - "title": "Extras - Recipes", - "section": "Looking at the predictors", - "text": "Looking at the predictors\n\nforested_train\n#> # A tibble: 8,749 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 1997 66 82 10 Tree 12.2 1315 18.4 1.88 24.9 11.8 121 1888 66 -84.9 32.4\n#> 2 No 1997 284 -99 58 Tree 10.3 1236 16.1 -0.26 22.3 9.92 68 1586 80 -85.0 34.1\n#> 3 Yes 2022 130 86 15 Tree 11.8 1194 17.6 1.3 24.1 11.2 61 1753 96 -83.0 33.1\n#> 4 Yes 2021 202 -55 3 Tree 10.7 1235 16.6 0.05 23.0 10.2 72 1682 65 -83.0 33.9\n#> 5 Yes 1995 75 -89 1 Tree 13.8 1256 19.2 3.63 25.5 12.9 57 1796 88 -83.4 31.3\n#> 6 No 1995 110 -53 5 Tree 12.4 1236 18.6 2.53 24.8 12.4 102 1835 51 -83.9 32.2\n#> 7 Yes 2022 111 73 12 Tree 11.5 1168 17.4 1 24.0 10.9 67 1772 84 -82.2 33.6\n#> 8 Yes 1997 230 96 14 Tree 9.98 1373 15.4 -1.35 21.8 9.03 46 1552 68 -85.3 34.8\n#> 9 Yes 2002 160 -88 13 Tree 11.1 1219 16.9 0.07 23.6 10.2 53 1731 95 -83.2 33.6\n#> 10 Yes 2020 39 9 6 Tree 13.9 1237 19.2 3.25 25.6 12.8 58 1812 86 -82.0 31.7\n#> # ℹ 8,739 more rows\n#> # ℹ 2 more variables: land_type <fct>, county <fct>" + "objectID": "slides/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", + "href": "slides/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", + "title": "Extras - workflowsets", + "section": "How can we compare multiple model workflows at once?", + "text": "How can we compare multiple model workflows at once?" }, { - "objectID": "slides/intro-extra-recipes.html#working-with-other-models", - "href": "slides/intro-extra-recipes.html#working-with-other-models", - "title": "Extras - Recipes", - "section": "Working with other models", - "text": "Working with other models\nSome models can’t handle non-numeric data\n\nLinear Regression\nK Nearest Neighbors\n\n\n\nSome models struggle if numeric predictors aren’t scaled\n\nK Nearest Neighbors\nAnything using gradient descent" + "objectID": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set", + "href": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nwf_set <- workflow_set(list(forested ~ .), list(tree_spec, rf_spec))\nwf_set\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" }, { - "objectID": "slides/intro-extra-recipes.html#types-of-needed-preprocessing", - "href": "slides/intro-extra-recipes.html#types-of-needed-preprocessing", - "title": "Extras - Recipes", - "section": "Types of needed preprocessing", - "text": "Types of needed preprocessing\n\nDo qualitative predictors require a numeric encoding?\nShould columns with a single unique value be removed?\nDoes the model struggle with missing data?\nDoes the model struggle with correlated predictors?\nShould predictors be centered and scaled?\nIs it helpful to transform predictors to be more symmetric?\n\n\nhttps://www.tmwr.org/pre-proc-table.html" + "objectID": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", + "href": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nwf_set_fit <- wf_set |>\n workflow_map(\"fit_resamples\", resamples = forested_folds)\nwf_set_fit\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" }, { - "objectID": "slides/intro-extra-recipes.html#two-types-of-preprocessing", - "href": "slides/intro-extra-recipes.html#two-types-of-preprocessing", - "title": "Extras - Recipes", - "section": "Two types of preprocessing", - "text": "Two types of preprocessing" + "objectID": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", + "href": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nwf_set_fit |>\n rank_results()\n#> # A tibble: 6 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_for… pre0_m… accura… 0.753 0.00462 10 formula rand… 1\n#> 2 formula_rand_for… pre0_m… brier_… 0.167 0.00312 10 formula rand… 1\n#> 3 formula_rand_for… pre0_m… roc_auc 0.757 0.00994 10 formula rand… 1\n#> 4 formula_decision… pre0_m… accura… 0.704 0.00653 10 formula deci… 2\n#> 5 formula_decision… pre0_m… brier_… 0.214 0.00349 10 formula deci… 2\n#> 6 formula_decision… pre0_m… roc_auc 0.692 0.00496 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" }, { - "objectID": "slides/intro-extra-recipes.html#two-types-of-preprocessing-1", - "href": "slides/intro-extra-recipes.html#two-types-of-preprocessing-1", - "title": "Extras - Recipes", - "section": "Two types of preprocessing", - "text": "Two types of preprocessing" + "objectID": "slides/intro-extra-workflowsets.html#your-turn", + "href": "slides/intro-extra-workflowsets.html#your-turn", + "title": "Extras - workflowsets", + "section": "Your turn", + "text": "Your turn\n\nWhen do you think a workflow set would be useful?\nDiscuss with your neighbors!\n\n\n\n−+\n03:00" }, { - "objectID": "slides/intro-extra-recipes.html#general-definitions", - "href": "slides/intro-extra-recipes.html#general-definitions", - "title": "Extras - Recipes", - "section": "General definitions", - "text": "General definitions\n\nData preprocessing is what you do to make your model successful.\nFeature engineering is what you do to the original predictors to make the model do the least work to perform great." + "objectID": "slides/intro-06-wrapping-up.html#your-turn", + "href": "slides/intro-06-wrapping-up.html#your-turn", + "title": "6 - Wrapping up", + "section": "Your turn", + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-extra-recipes.html#working-with-dates", - "href": "slides/intro-extra-recipes.html#working-with-dates", - "title": "Extras - Recipes", - "section": "Working with dates", - "text": "Working with dates\nDatetime variables are automatically converted to an integer if given as a raw predictor. To avoid this, it can be re-encoded as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nLeap year\nIndicators for holidays" + "objectID": "slides/intro-06-wrapping-up.html#resources-to-keep-learning", + "href": "slides/intro-06-wrapping-up.html#resources-to-keep-learning", + "title": "6 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://tidymodels.aml4td.org/\n\n\n\n\nhttps://smltar.com/\n\n\n\n\nhttps://feaz-book.com/\n\n\n\nFollow us on Bluesky, Mastodon and at the tidyverse blog for updates!" }, { - "objectID": "slides/intro-extra-recipes.html#two-types-of-transformations", - "href": "slides/intro-extra-recipes.html#two-types-of-transformations", - "title": "Extras - Recipes", - "section": "Two types of transformations", - "text": "Two types of transformations\n\n\n\nStatic\n\nSquare root, log, inverse\nDummies for known levels\nDate time extractions\n\n\nTrained\n\nCentering & scaling\nImputation\nPCA\nAnything for unknown factor levels\n\n\n\nTrained methods need to calculate sufficient information to be applied again." + "objectID": "slides/intro-04-evaluating-models.html#looking-at-predictions", + "href": "slides/intro-04-evaluating-models.html#looking-at-predictions", + "title": "4 - Evaluating models", + "section": "Looking at predictions", + "text": "Looking at predictions\n\naugment(forested_fit, new_data = forested_train)\n#> # A tibble: 8,749 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness roughness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 0.931 0.0690 Yes 1997 66 82 10\n#> 2 Yes 0.983 0.0172 No 1997 284 -99 58\n#> 3 Yes 0.960 0.0401 Yes 2022 130 86 15\n#> 4 Yes 0.870 0.130 Yes 2021 202 -55 3\n#> 5 Yes 0.823 0.177 Yes 1995 75 -89 1\n#> 6 Yes 0.758 0.242 No 1995 110 -53 5\n#> 7 Yes 0.823 0.177 Yes 2022 111 73 12\n#> 8 No 0.467 0.533 Yes 1997 230 96 14\n#> 9 Yes 0.983 0.0172 Yes 2002 160 -88 13\n#> 10 Yes 0.871 0.129 Yes 2020 39 9 6\n#> # ℹ 8,739 more rows\n#> # ℹ 14 more variables: tree_no_tree <fct>, dew_temp <dbl>, precip_annual <dbl>,\n#> # temp_annual_mean <dbl>, temp_annual_min <dbl>, temp_annual_max <dbl>,\n#> # temp_january_min <dbl>, vapor_min <dbl>, vapor_max <dbl>,\n#> # canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>, county <fct>" }, { - "objectID": "slides/intro-extra-recipes.html#the-recipes-package", - "href": "slides/intro-extra-recipes.html#the-recipes-package", - "title": "Extras - Recipes", - "section": "The recipes package", - "text": "The recipes package\n\n\nModular + extensible\nWorks well with pipes ,|> and %>%\nDeferred evaluation\nIsolates test data from training data\nCan do things formulas can’t" + "objectID": "slides/intro-04-evaluating-models.html#confusion-matrix", + "href": "slides/intro-04-evaluating-models.html#confusion-matrix", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix" }, { - "objectID": "slides/intro-extra-recipes.html#how-to-write-a-recipe", - "href": "slides/intro-extra-recipes.html#how-to-write-a-recipe", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())" + "objectID": "slides/intro-04-evaluating-models.html#confusion-matrix-1", + "href": "slides/intro-04-evaluating-models.html#confusion-matrix-1", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) |>\n conf_mat(truth = forested, estimate = .pred_class)\n#> Truth\n#> Prediction Yes No\n#> Yes 5884 869\n#> No 438 1558" }, { - "objectID": "slides/intro-extra-recipes.html#how-to-write-a-recipe-1", - "href": "slides/intro-extra-recipes.html#how-to-write-a-recipe-1", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())\n\n\nStart by calling recipe() to denote the data source and variables used." + "objectID": "slides/intro-04-evaluating-models.html#confusion-matrix-2", + "href": "slides/intro-04-evaluating-models.html#confusion-matrix-2", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) |>\n conf_mat(truth = forested, estimate = .pred_class) |>\n autoplot(type = \"heatmap\")" }, { - "objectID": "slides/intro-extra-recipes.html#how-to-write-a-recipe-2", - "href": "slides/intro-extra-recipes.html#how-to-write-a-recipe-2", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())\n\n\nSpecify what actions to take by adding step_*()s." + "objectID": "slides/intro-04-evaluating-models.html#your-turn", + "href": "slides/intro-04-evaluating-models.html#your-turn", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\n\nHow would you combine TP, TN, FP, and FN into one number to describe how often the model is correct?\n\n\n\n\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-extra-recipes.html#how-to-write-a-recipe-3", - "href": "slides/intro-extra-recipes.html#how-to-write-a-recipe-3", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>   step_normalize(all_numeric_predictors())\n\n\nUse {tidyselect} and recipes-specific selectors to denote affected variables." + "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance", + "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851" }, { - "objectID": "slides/intro-extra-recipes.html#using-a-recipe", - "href": "slides/intro-extra-recipes.html#using-a-recipe", - "title": "Extras - Recipes", - "section": "Using a recipe", - "text": "Using a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>   step_normalize(all_numeric_predictors())\n\n\nSave the recipe we like so that we can use it in various places, e.g., with different models." + "objectID": "slides/intro-04-evaluating-models.html#dangers-of-accuracy", + "href": "slides/intro-04-evaluating-models.html#dangers-of-accuracy", + "title": "4 - Evaluating models", + "section": "Dangers of accuracy ", + "text": "Dangers of accuracy \nWe need to be careful of using accuracy() since it can give “good” performance by only predicting one way with imbalanced data:\n\naugment(forested_fit, new_data = forested_train) %>%\n mutate(.pred_class = factor(\"Yes\", levels = c(\"Yes\", \"No\"))) %>%\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.723" }, { - "objectID": "slides/intro-extra-recipes.html#using-a-recipe-with-workflows", - "href": "slides/intro-extra-recipes.html#using-a-recipe-with-workflows", - "title": "Extras - Recipes", - "section": "Using a recipe with workflows", - "text": "Using a recipe with workflows\nRecipes are typically combined with a model in a workflow() object:\n\n\nforested_wflow <- workflow() |>\n  add_recipe(forested_rec) |>\n  add_model(linear_reg())" + "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-1", + "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-1", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.931" }, { - "objectID": "slides/intro-extra-recipes.html#recipes-are-estimated", - "href": "slides/intro-extra-recipes.html#recipes-are-estimated", - "title": "Extras - Recipes", - "section": "Recipes are estimated", - "text": "Recipes are estimated\nEvery preprocessing step in a recipe that involved calculations uses the training set. For example:\n\nLevels of a factor\nDetermination of zero-variance\nNormalization\nFeature extraction\n\nOnce a recipe is added to a workflow, this occurs when fit() is called." + "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-2", + "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-2", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.931\n\n\n\naugment(forested_fit, new_data = forested_train) |>\n specificity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.642" }, { - "objectID": "slides/intro-extra-recipes.html#debugging-a-recipe", - "href": "slides/intro-extra-recipes.html#debugging-a-recipe", - "title": "Extras - Recipes", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. forested_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4.\n\n\n\n\nAnother function, bake(), is analogous to predict(), and gives you the processed data back." + "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-3", + "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-3", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\nforested_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(forested_fit, new_data = forested_train) |>\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n#> 2 specificity binary 0.642\n#> 3 sensitivity binary 0.931" }, { - "objectID": "slides/intro-extra-recipes.html#your-turn", - "href": "slides/intro-extra-recipes.html#your-turn", - "title": "Extras - Recipes", - "section": "Your turn", - "text": "Your turn\n\n\nTake the recipe and prep() then bake() it to see what the resulting data set looks like.\nTry removing steps to see how the result changes.\n\n\n\n\n−+\n05:00" + "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-4", + "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-4", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \nMetrics and metric sets work with grouped data frames!\n\naugment(forested_fit, new_data = forested_train) |>\n group_by(tree_no_tree) |>\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 2 × 4\n#> tree_no_tree .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 Tree accuracy binary 0.875\n#> 2 No tree accuracy binary 0.807" }, { - "objectID": "slides/intro-extra-recipes.html#printing-a-recipe", - "href": "slides/intro-extra-recipes.html#printing-a-recipe", - "title": "Extras - Recipes", - "section": "Printing a recipe", - "text": "Printing a recipe\n\nforested_rec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Operations\n#> • Dummy variables from: all_nominal_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Log transformation on: canopy_cover\n#> • Centering and scaling for: all_numeric_predictors()" + "objectID": "slides/intro-04-evaluating-models.html#your-turn-1", + "href": "slides/intro-04-evaluating-models.html#your-turn-1", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nApply the forested_metrics metric set to augment()\noutput grouped by tree_no_tree.\nDo any metrics differ substantially between groups?\n\n\n\n−+\n05:00\n\n\n\n\nThe specificity for \"Tree\" is a good bit lower than it is for \"No tree\".\nSpecificity is the proportion of negatives that are correctly identified as negatives. “Negative” is the non-event level of the outcome, i.e. “non-forested.” So, when this index classifies the plot as having a tree, the model does not do well at correctly identifying the plot as non-forested when it is indeed non-forested." }, { - "objectID": "slides/intro-extra-recipes.html#prepping-a-recipe", - "href": "slides/intro-extra-recipes.html#prepping-a-recipe", - "title": "Extras - Recipes", - "section": "Prepping a recipe", - "text": "Prepping a recipe\n\nprep(forested_rec)\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Training information\n#> Training data contained 8749 data points and no incomplete rows.\n#> \n#> ── Operations\n#> • Dummy variables from: tree_no_tree, land_type, county | Trained\n#> • Zero variance filter removed: <none> | Trained\n#> • Log transformation on: canopy_cover | Trained\n#> • Centering and scaling for: year elevation, ... | Trained" + "objectID": "slides/intro-04-evaluating-models.html#two-class-data", + "href": "slides/intro-04-evaluating-models.html#two-class-data", + "title": "4 - Evaluating models", + "section": "Two class data", + "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" }, { - "objectID": "slides/intro-extra-recipes.html#baking-a-recipe", - "href": "slides/intro-extra-recipes.html#baking-a-recipe", - "title": "Extras - Recipes", - "section": "Baking a recipe", - "text": "Baking a recipe\n\nprep(forested_rec) |>\n bake(new_data = forested_train)\n#> # A tibble: 8,749 × 177\n#> year elevation eastness roughness dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat forested\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> \n#> 1 -1.15 -0.521 1.13 0.0463 -0.232 0.178 0.172 -0.107 0.338 -0.00909 3.10 1.01 0.407 -1.44 -0.0184 Yes \n#> 2 -1.15 1.10 -1.50 3.91 -1.39 -0.568 -1.31 -1.33 -1.28 -1.28 0.0408 -0.941 0.652 -1.51 1.36 No \n#> 3 0.892 -0.0440 1.19 0.449 -0.422 -0.965 -0.305 -0.438 -0.170 -0.438 -0.363 0.140 0.884 0.331 0.556 Yes \n#> 4 0.810 0.493 -0.858 -0.518 -1.11 -0.577 -1.01 -1.15 -0.878 -1.10 0.272 -0.320 0.388 0.331 1.26 Yes \n#> 5 -1.31 -0.454 -1.35 -0.679 0.748 -0.379 0.701 0.893 0.683 0.702 -0.594 0.418 0.773 -0.0643 -0.992 Yes \n#> 6 -1.31 -0.193 -0.829 -0.357 -0.0546 -0.568 0.331 0.264 0.281 0.366 2.00 0.670 0.0802 -0.468 -0.237 No \n#> 7 0.892 -0.186 1.00 0.207 -0.631 -1.21 -0.444 -0.609 -0.264 -0.619 -0.0169 0.262 0.714 1.01 0.929 Yes \n#> 8 -1.15 0.702 1.34 0.369 -1.56 0.726 -1.77 -1.95 -1.60 -1.87 -1.23 -1.16 0.445 -1.77 2.03 Yes \n#> 9 -0.740 0.180 -1.34 0.288 -0.858 -0.729 -0.801 -1.14 -0.515 -1.08 -0.825 -0.00273 0.870 0.122 0.988 Yes \n#> 10 0.729 -0.722 0.0719 -0.276 0.816 -0.559 0.708 0.676 0.745 0.635 -0.536 0.521 0.744 1.22 -0.653 Yes \n#> # ℹ 8,739 more rows\n#> # ℹ 161 more variables: tree_no_tree_No.tree <dbl>, land_type_Non.tree.vegetation <dbl>, land_type_Tree <dbl>, county_Atkinson <dbl>, county_Bacon <dbl>, county_Baker <dbl>, county_Baldwin <dbl>,\n#> # county_Banks <dbl>, county_Barrow <dbl>, county_Bartow <dbl>, county_Ben.Hill <dbl>, county_Berrien <dbl>, county_Bibb <dbl>, county_Bleckley <dbl>, county_Brantley <dbl>, county_Brooks <dbl>,\n#> # county_Bryan <dbl>, county_Bulloch <dbl>, county_Burke <dbl>, county_Butts <dbl>, county_Calhoun <dbl>, county_Camden <dbl>, county_Candler <dbl>, county_Carroll <dbl>, county_Catoosa <dbl>,\n#> # county_Charlton <dbl>, county_Chatham <dbl>, county_Chattahoochee <dbl>, county_Chattooga <dbl>, county_Cherokee <dbl>, county_Clarke <dbl>, county_Clay <dbl>, county_Clayton <dbl>,\n#> # county_Clinch <dbl>, county_Cobb <dbl>, county_Coffee <dbl>, county_Colquitt <dbl>, county_Columbia <dbl>, county_Cook <dbl>, county_Coweta <dbl>, county_Crawford <dbl>, county_Crisp <dbl>,\n#> # county_Dade <dbl>, county_Dawson <dbl>, county_Decatur <dbl>, county_DeKalb <dbl>, county_Dodge <dbl>, county_Dooly <dbl>, county_Dougherty <dbl>, county_Douglas <dbl>, county_Early <dbl>, …" + "objectID": "slides/intro-04-evaluating-models.html#varying-the-threshold", + "href": "slides/intro-04-evaluating-models.html#varying-the-threshold", + "title": "4 - Evaluating models", + "section": "Varying the threshold", + "text": "Varying the threshold" }, { - "objectID": "slides/intro-extra-recipes.html#tidying-a-recipe", - "href": "slides/intro-extra-recipes.html#tidying-a-recipe", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\nOnce a recipe as been estimated, there are various bits of information saved in it.\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "slides/intro-04-evaluating-models.html#roc-curves", + "href": "slides/intro-04-evaluating-models.html#roc-curves", + "title": "4 - Evaluating models", + "section": "ROC curves", + "text": "ROC curves\n\n\nFor an ROC (receiver operator characteristic) curve, we plot\n\nthe false positive rate (1 - specificity) on the x-axis\nthe true positive rate (sensitivity) on the y-axis\n\nwith sensitivity and specificity calculated at all possible thresholds.\n\n\n\n\n\n\n\n\n\n\n\n\nROC curves are insensitive to class imbalance." }, { - "objectID": "slides/intro-extra-recipes.html#your-turn-1", - "href": "slides/intro-extra-recipes.html#your-turn-1", - "title": "Extras - Recipes", - "section": "Your turn", - "text": "Your turn\n\nTake a prepped recipe and use the tidy() function on it.\nUse the number argument to inspect different steps.\n\n\n\n\n−+\n05:00" + "objectID": "slides/intro-04-evaluating-models.html#roc-curves-1", + "href": "slides/intro-04-evaluating-models.html#roc-curves-1", + "title": "4 - Evaluating models", + "section": "ROC curves", + "text": "ROC curves\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢" }, { - "objectID": "slides/intro-extra-recipes.html#tidying-a-recipe-1", - "href": "slides/intro-extra-recipes.html#tidying-a-recipe-1", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\n\nprep(forested_rec) |>\n tidy()\n#> # A tibble: 4 × 6\n#> number operation type trained skip id \n#> <int> <chr> <chr> <lgl> <lgl> <chr> \n#> 1 1 step dummy TRUE FALSE dummy_hIEnQ \n#> 2 2 step zv TRUE FALSE zv_ZrQBx \n#> 3 3 step log TRUE FALSE log_6es7X \n#> 4 4 step normalize TRUE FALSE normalize_XsIxb" + "objectID": "slides/intro-04-evaluating-models.html#roc-curves-2", + "href": "slides/intro-04-evaluating-models.html#roc-curves-2", + "title": "4 - Evaluating models", + "section": "ROC curves ", + "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(forested_fit, new_data = forested_train) |> \n roc_curve(truth = forested, .pred_Yes) |>\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.143 0.267 0.990\n#> 3 0.385 0.571 0.951\n\naugment(forested_fit, new_data = forested_train) |> \n roc_auc(truth = forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.881" }, { - "objectID": "slides/intro-extra-recipes.html#tidying-a-recipe-2", - "href": "slides/intro-extra-recipes.html#tidying-a-recipe-2", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\n\nprep(forested_rec) |>\n tidy(number = 1)\n#> # A tibble: 161 × 3\n#> terms columns id \n#> <chr> <chr> <chr> \n#> 1 tree_no_tree No tree dummy_hIEnQ\n#> 2 land_type Non-tree vegetation dummy_hIEnQ\n#> 3 land_type Tree dummy_hIEnQ\n#> 4 county Atkinson dummy_hIEnQ\n#> 5 county Bacon dummy_hIEnQ\n#> 6 county Baker dummy_hIEnQ\n#> 7 county Baldwin dummy_hIEnQ\n#> 8 county Banks dummy_hIEnQ\n#> 9 county Barrow dummy_hIEnQ\n#> 10 county Bartow dummy_hIEnQ\n#> # ℹ 151 more rows" + "objectID": "slides/intro-04-evaluating-models.html#roc-curve-plot", + "href": "slides/intro-04-evaluating-models.html#roc-curve-plot", + "title": "4 - Evaluating models", + "section": "ROC curve plot ", + "text": "ROC curve plot \n\n\naugment(forested_fit, \n new_data = forested_train) |> \n roc_curve(truth = forested, \n .pred_Yes) |>\n autoplot()" }, { - "objectID": "slides/intro-extra-recipes.html#using-a-recipe-in-tidymodels", - "href": "slides/intro-extra-recipes.html#using-a-recipe-in-tidymodels", - "title": "Extras - Recipes", - "section": "Using a recipe in tidymodels", - "text": "Using a recipe in tidymodels\nThe recommended way to use a recipe in tidymodels is to use it as part of a workflow().\n\nforested_wflow <- workflow() |> \n add_recipe(forested_rec) |> \n add_model(linear_reg())\n\nWhen used in this way, you don’t need to worry about prep() and bake() as it is handled for you." + "objectID": "slides/intro-04-evaluating-models.html#your-turn-2", + "href": "slides/intro-04-evaluating-models.html#your-turn-2", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-extra-recipes.html#more-information", - "href": "slides/intro-extra-recipes.html#more-information", - "title": "Extras - Recipes", - "section": "More information", - "text": "More information\n\nhttps://recipes.tidymodels.org/\nhttps://www.tmwr.org/recipes.html" + "objectID": "slides/intro-04-evaluating-models.html#brier-score", + "href": "slides/intro-04-evaluating-models.html#brier-score", + "title": "4 - Evaluating models", + "section": "Brier score", + "text": "Brier score\nWhat if we don’t turn predicted probabilities into class predictions?\n\nThe Brier score is analogous to the mean squared error in regression models:\n\\[\nBrier_{class} = \\frac{1}{NC}\\sum_{i=1}^N\\sum_{k=1}^C (y_{ik} - \\hat{p}_{ik})^2\n\\]" }, { - "objectID": "slides/intro-05-tuning-models.html#tuning-parameters", - "href": "slides/intro-05-tuning-models.html#tuning-parameters", - "title": "5 - Tuning models", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "slides/intro-04-evaluating-models.html#brier-score-1", + "href": "slides/intro-04-evaluating-models.html#brier-score-1", + "title": "4 - Evaluating models", + "section": "Brier score", + "text": "Brier score\n\naugment(forested_fit, new_data = forested_train) |> \n brier_class(truth = forested, .pred_Yes) \n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.113\n\n\nSmaller values are better, for binary classification the “bad model threshold” is about 0.25." }, { - "objectID": "slides/intro-05-tuning-models.html#optimize-tuning-parameters", - "href": "slides/intro-05-tuning-models.html#optimize-tuning-parameters", - "title": "5 - Tuning models", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "slides/intro-04-evaluating-models.html#separation-vs-calibration", + "href": "slides/intro-04-evaluating-models.html#separation-vs-calibration", + "title": "4 - Evaluating models", + "section": "Separation vs calibration", + "text": "Separation vs calibration\n\n\nThe ROC captures separation.\n\n\n\n\n\n\n\n\n\n\nThe Brier score captures calibration.\n\n\n\n\n\n\n\n\n\n\n\n\nGood separation: the densities don’t overlap.\nGood calibration: the calibration line follows the diagonal.\n\nCalibration plot: We bin observations according to predicted probability. In the bin for 20%-30% predicted prob, we should see an event rate of ~25% if the model is well-calibrated." }, { - "objectID": "slides/intro-05-tuning-models.html#optimize-tuning-parameters-1", - "href": "slides/intro-05-tuning-models.html#optimize-tuning-parameters-1", - "title": "5 - Tuning models", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-1", + "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-1", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting", + "text": "Dangers of overfitting" }, { - "objectID": "slides/intro-05-tuning-models.html#specifying-tuning-parameters", - "href": "slides/intro-05-tuning-models.html#specifying-tuning-parameters", - "title": "5 - Tuning models", - "section": "Specifying tuning parameters", - "text": "Specifying tuning parameters\nLet’s take our previous random forest workflow and tag for tuning the minimum number of data points in each node:\n\nrf_spec <- rand_forest(min_n = tune()) |> \n set_mode(\"classification\")\n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" + "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-2", + "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-2", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "slides/intro-05-tuning-models.html#try-out-multiple-values", - "href": "slides/intro-05-tuning-models.html#try-out-multiple-values", - "title": "5 - Tuning models", - "section": "Try out multiple values", - "text": "Try out multiple values\ntune_grid() works similar to fit_resamples() but covers multiple parameter values:\n\nset.seed(22)\nrf_res <- tune_grid(\n rf_wflow,\n forested_folds,\n grid = 5\n)" + "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-3", + "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-3", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\nforested_fit |>\n augment(forested_train)\n#> # A tibble: 8,749 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness roughness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 0.931 0.0690 Yes 1997 66 82 10\n#> 2 Yes 0.983 0.0172 No 1997 284 -99 58\n#> 3 Yes 0.960 0.0401 Yes 2022 130 86 15\n#> 4 Yes 0.870 0.130 Yes 2021 202 -55 3\n#> 5 Yes 0.823 0.177 Yes 1995 75 -89 1\n#> 6 Yes 0.758 0.242 No 1995 110 -53 5\n#> 7 Yes 0.823 0.177 Yes 2022 111 73 12\n#> 8 No 0.467 0.533 Yes 1997 230 96 14\n#> 9 Yes 0.983 0.0172 Yes 2002 160 -88 13\n#> 10 Yes 0.871 0.129 Yes 2020 39 9 6\n#> # ℹ 8,739 more rows\n#> # ℹ 14 more variables: tree_no_tree <fct>, dew_temp <dbl>, precip_annual <dbl>,\n#> # temp_annual_mean <dbl>, temp_annual_min <dbl>, temp_annual_max <dbl>,\n#> # temp_january_min <dbl>, vapor_min <dbl>, vapor_max <dbl>,\n#> # canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>, county <fct>\n\nWe call this “resubstitution” or “repredicting the training set”" }, { - "objectID": "slides/intro-05-tuning-models.html#compare-results", - "href": "slides/intro-05-tuning-models.html#compare-results", - "title": "5 - Tuning models", - "section": "Compare results", - "text": "Compare results\nInspecting results and selecting the best-performing hyperparameter(s):\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 40 roc_auc binary 0.762 10 0.0103 pre0_mod5_post0\n#> 2 31 roc_auc binary 0.761 10 0.0100 pre0_mod4_post0\n#> 3 22 roc_auc binary 0.760 10 0.00997 pre0_mod3_post0\n#> 4 12 roc_auc binary 0.758 10 0.0102 pre0_mod2_post0\n#> 5 3 roc_auc binary 0.755 10 0.00980 pre0_mod1_post0\n\nbest_parameter <- select_best(rf_res)\nbest_parameter\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 40 pre0_mod5_post0\n\ncollect_metrics() and autoplot() are also available." + "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-4", + "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-4", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n\nWe call this a “resubstitution estimate”" }, { - "objectID": "slides/intro-05-tuning-models.html#the-final-fit", - "href": "slides/intro-05-tuning-models.html#the-final-fit", - "title": "5 - Tuning models", - "section": "The final fit", - "text": "The final fit\n\nrf_wflow <- finalize_workflow(rf_wflow, best_parameter)\n\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.764 pre0_mod0_post0\n#> 2 roc_auc binary 0.764 pre0_mod0_post0\n#> 3 brier_class binary 0.161 pre0_mod0_post0" + "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-5", + "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-5", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851" }, { - "objectID": "slides/intro-05-tuning-models.html#your-turn", - "href": "slides/intro-05-tuning-models.html#your-turn", - "title": "5 - Tuning models", - "section": "Your turn", - "text": "Your turn\n\nModify your model workflow to tune one or more parameters.\nUse grid search to find the best parameter(s).\n\n\n\n−+\n05:00" + "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-6", + "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-6", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n\n\n\nforested_fit |>\n augment(forested_test) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.711\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#your-turn", - "href": "slides/intro-03-what-makes-a-model.html#your-turn", - "title": "3 - What makes a model?", + "objectID": "slides/intro-04-evaluating-models.html#your-turn-3", + "href": "slides/intro-04-evaluating-models.html#your-turn-3", + "title": "4 - Evaluating models", "section": "Your turn", - "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets\nbrulee for regression using torch" - }, - { - "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model", - "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\n\nChoose a model type\nSpecify an engine\nSet the mode" + "text": "Your turn\n\nUse augment() and a metric function to compute a classification metric like roc_auc().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-1", - "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-1", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" + "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-7", + "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-7", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n roc_auc(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.881\n\n\n\nforested_fit |>\n augment(forested_test) |>\n roc_auc(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.703\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-2", - "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-2", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model type\nSpecify an engine\nSet the mode" + "objectID": "slides/intro-04-evaluating-models.html#cross-validation", + "href": "slides/intro-04-evaluating-models.html#cross-validation", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-3", - "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-3", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() |>\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" + "objectID": "slides/intro-04-evaluating-models.html#cross-validation-1", + "href": "slides/intro-04-evaluating-models.html#cross-validation-1", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-4", - "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-4", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() |>\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" + "objectID": "slides/intro-04-evaluating-models.html#your-turn-4", + "href": "slides/intro-04-evaluating-models.html#your-turn-4", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-5", - "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-5", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model type\nSpecify an engine\nSet the mode" + "objectID": "slides/intro-04-evaluating-models.html#cross-validation-2", + "href": "slides/intro-04-evaluating-models.html#cross-validation-2", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(forested_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7874/875]> Fold01\n#> 2 <split [7874/875]> Fold02\n#> 3 <split [7874/875]> Fold03\n#> 4 <split [7874/875]> Fold04\n#> 5 <split [7874/875]> Fold05\n#> 6 <split [7874/875]> Fold06\n#> 7 <split [7874/875]> Fold07\n#> 8 <split [7874/875]> Fold08\n#> 9 <split [7874/875]> Fold09\n#> 10 <split [7875/874]> Fold10" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-6", - "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-6", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" + "objectID": "slides/intro-04-evaluating-models.html#cross-validation-3", + "href": "slides/intro-04-evaluating-models.html#cross-validation-3", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWhat is in this?\n\nforested_folds <- vfold_cv(forested_train)\nforested_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n\n\nTalk about a list column, storing non-atomic types in dataframe" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-7", - "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-7", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree() |> \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" + "objectID": "slides/intro-04-evaluating-models.html#cross-validation-4", + "href": "slides/intro-04-evaluating-models.html#cross-validation-4", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(forested_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [6999/1750]> Fold1\n#> 2 <split [6999/1750]> Fold2\n#> 3 <split [6999/1750]> Fold3\n#> 4 <split [6999/1750]> Fold4\n#> 5 <split [7000/1749]> Fold5" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-8", - "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-8", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model type\nSpecify an engine\nSet the mode" + "objectID": "slides/intro-04-evaluating-models.html#cross-validation-5", + "href": "slides/intro-04-evaluating-models.html#cross-validation-5", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nforested_folds <- vfold_cv(forested_train, v = 10)\nforested_folds\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7874/875]> Fold01\n#> 2 <split [7874/875]> Fold02\n#> 3 <split [7874/875]> Fold03\n#> 4 <split [7874/875]> Fold04\n#> 5 <split [7874/875]> Fold05\n#> 6 <split [7874/875]> Fold06\n#> 7 <split [7874/875]> Fold07\n#> 8 <split [7874/875]> Fold08\n#> 9 <split [7874/875]> Fold09\n#> 10 <split [7875/874]> Fold10\n\n\nSet the seed when creating resamples" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-1", - "href": "slides/intro-03-what-makes-a-model.html#your-turn-1", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a logistic regression model.\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/\n\n\nExtension/Challenge: Edit this code to use a different model. For example, try using a conditional inference tree as implemented in the partykit package by changing the engine - or try an entirely different model type!\n\n\n\n−+\n05:00" + "objectID": "slides/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", + "href": "slides/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", + "title": "4 - Evaluating models", + "section": "Fit our model to the resamples", + "text": "Fit our model to the resamples\n\nforested_res <- fit_resamples(forested_wflow, forested_folds)\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 4]>" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#models-well-be-using-today", - "href": "slides/intro-03-what-makes-a-model.html#models-well-be-using-today", - "title": "3 - What makes a model?", - "section": "Models we’ll be using today", - "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" + "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance", + "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nforested_res |>\n collect_metrics()\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.704 10 0.00653 pre0_mod0_post0\n#> 2 brier_class binary 0.214 10 0.00349 pre0_mod0_post0\n#> 3 roc_auc binary 0.692 10 0.00496 pre0_mod0_post0\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nWe can reliably measure performance using only the training data 🎉" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#logistic-regression", - "href": "slides/intro-03-what-makes-a-model.html#logistic-regression", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" + "objectID": "slides/intro-04-evaluating-models.html#comparing-metrics", + "href": "slides/intro-04-evaluating-models.html#comparing-metrics", + "title": "4 - Evaluating models", + "section": "Comparing metrics ", + "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\nforested_res |>\n collect_metrics() |> \n select(.metric, mean, n)\n#> # A tibble: 3 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.704 10\n#> 2 brier_class 0.214 10\n#> 3 roc_auc 0.692 10\n\n\nThe ROC AUC previously was\n\n0.88 for the training set\n0.7 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#logistic-regression-1", - "href": "slides/intro-03-what-makes-a-model.html#logistic-regression-1", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" + "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance-1", + "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance-1", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_forested <- control_resamples(save_pred = TRUE)\nforested_res <- fit_resamples(forested_wflow, forested_folds, control = ctrl_forested)\n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 4]> <tibble>" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#logistic-regression-2", - "href": "slides/intro-03-what-makes-a-model.html#logistic-regression-2", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nFind a sigmoid line that separates the two classes" + "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance-2", + "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance-2", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nforested_preds <- collect_predictions(forested_res)\nforested_preds\n#> # A tibble: 8,749 × 7\n#> .pred_class .pred_Yes .pred_No id forested .row .config \n#> <fct> <dbl> <dbl> <chr> <fct> <int> <chr> \n#> 1 Yes 0.953 0.0473 Fold01 No 2 pre0_mod0_post0\n#> 2 Yes 0.6 0.4 Fold01 No 6 pre0_mod0_post0\n#> 3 Yes 0.848 0.152 Fold01 Yes 7 pre0_mod0_post0\n#> 4 Yes 0.941 0.0588 Fold01 Yes 36 pre0_mod0_post0\n#> 5 Yes 0.895 0.105 Fold01 No 38 pre0_mod0_post0\n#> 6 Yes 1 0 Fold01 Yes 58 pre0_mod0_post0\n#> 7 No 0.187 0.812 Fold01 No 69 pre0_mod0_post0\n#> 8 Yes 0.905 0.0952 Fold01 Yes 71 pre0_mod0_post0\n#> 9 Yes 0.907 0.0930 Fold01 Yes 74 pre0_mod0_post0\n#> 10 Yes 0.904 0.0962 Fold01 Yes 80 pre0_mod0_post0\n#> # ℹ 8,739 more rows" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#decision-trees", - "href": "slides/intro-03-what-makes-a-model.html#decision-trees", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance-3", + "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance-3", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nforested_preds |> \n group_by(id) |>\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.729\n#> 2 Fold02 accuracy binary 0.685\n#> 3 Fold03 accuracy binary 0.687\n#> 4 Fold04 accuracy binary 0.711\n#> 5 Fold05 accuracy binary 0.736\n#> 6 Fold06 accuracy binary 0.670\n#> 7 Fold07 accuracy binary 0.699\n#> 8 Fold08 accuracy binary 0.710\n#> 9 Fold09 accuracy binary 0.699\n#> 10 Fold10 accuracy binary 0.719\n#> # ℹ 20 more rows" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#decision-trees-1", - "href": "slides/intro-03-what-makes-a-model.html#decision-trees-1", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" + "objectID": "slides/intro-04-evaluating-models.html#where-are-the-fitted-models", + "href": "slides/intro-04-evaluating-models.html#where-are-the-fitted-models", + "title": "4 - Evaluating models", + "section": "Where are the fitted models? ", + "text": "Where are the fitted models? \n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 4]> <tibble>\n\n\n🗑️" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#decision-trees-2", - "href": "slides/intro-03-what-makes-a-model.html#decision-trees-2", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "slides/intro-04-evaluating-models.html#bootstrapping", + "href": "slides/intro-04-evaluating-models.html#bootstrapping", + "title": "4 - Evaluating models", + "section": "Bootstrapping", + "text": "Bootstrapping" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "href": "slides/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "title": "3 - What makes a model?", - "section": "All models are wrong, but some are useful!", - "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" + "objectID": "slides/intro-04-evaluating-models.html#bootstrapping-1", + "href": "slides/intro-04-evaluating-models.html#bootstrapping-1", + "title": "4 - Evaluating models", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(forested_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [8749/3218]> Bootstrap01\n#> 2 <split [8749/3264]> Bootstrap02\n#> 3 <split [8749/3220]> Bootstrap03\n#> 4 <split [8749/3208]> Bootstrap04\n#> 5 <split [8749/3230]> Bootstrap05\n#> 6 <split [8749/3197]> Bootstrap06\n#> 7 <split [8749/3193]> Bootstrap07\n#> 8 <split [8749/3226]> Bootstrap08\n#> 9 <split [8749/3243]> Bootstrap09\n#> 10 <split [8749/3233]> Bootstrap10\n#> # ℹ 15 more rows" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "href": "slides/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "title": "3 - What makes a model?", - "section": "Workflows bind preprocessors and models", - "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" + "objectID": "slides/intro-04-evaluating-models.html#the-whole-game---status-update", + "href": "slides/intro-04-evaluating-models.html#the-whole-game---status-update", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#what-is-wrong-with-this", - "href": "slides/intro-03-what-makes-a-model.html#what-is-wrong-with-this", - "title": "3 - What makes a model?", - "section": "What is wrong with this?", - "text": "What is wrong with this?" + "objectID": "slides/intro-04-evaluating-models.html#your-turn-5", + "href": "slides/intro-04-evaluating-models.html#your-turn-5", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the functions)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#why-a-workflow", - "href": "slides/intro-03-what-makes-a-model.html#why-a-workflow", - "title": "3 - What makes a model?", - "section": "Why a workflow()? ", - "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on this in Getting More Out of Feature Engineering and Tuning for Machine Learning)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" + "objectID": "slides/intro-04-evaluating-models.html#monte-carlo-cross-validation", + "href": "slides/intro-04-evaluating-models.html#monte-carlo-cross-validation", + "title": "4 - Evaluating models", + "section": "Monte Carlo Cross-Validation ", + "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(forested_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6561/2188]> Resample01\n#> 2 <split [6561/2188]> Resample02\n#> 3 <split [6561/2188]> Resample03\n#> 4 <split [6561/2188]> Resample04\n#> 5 <split [6561/2188]> Resample05\n#> 6 <split [6561/2188]> Resample06\n#> 7 <split [6561/2188]> Resample07\n#> 8 <split [6561/2188]> Resample08\n#> 9 <split [6561/2188]> Resample09\n#> 10 <split [6561/2188]> Resample10" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#a-model-workflow-1", - "href": "slides/intro-03-what-makes-a-model.html#a-model-workflow-1", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\ntree_spec |> \n fit(forested ~ ., data = forested_train) \n#> parsnip model object\n#> \n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" + "objectID": "slides/intro-04-evaluating-models.html#validation-set", + "href": "slides/intro-04-evaluating-models.html#validation-set", + "title": "4 - Evaluating models", + "section": "Validation set ", + "text": "Validation set \n\nset.seed(853)\nforested_val_split <- initial_validation_split(forested)\nvalidation_set(forested_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [4264/1421]> validation\n\n\nA validation set is just another type of resample" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#a-model-workflow-2", - "href": "slides/intro-03-what-makes-a-model.html#a-model-workflow-2", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\nworkflow() |>\n add_formula(forested ~ .) |>\n add_model(tree_spec) |>\n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" + "objectID": "slides/intro-04-evaluating-models.html#random-forest-1", + "href": "slides/intro-04-evaluating-models.html#random-forest-1", + "title": "4 - Evaluating models", + "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this later!), as long as there are enough trees" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#a-model-workflow-3", - "href": "slides/intro-03-what-makes-a-model.html#a-model-workflow-3", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\nworkflow(forested ~ ., tree_spec) |> \n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" + "objectID": "slides/intro-04-evaluating-models.html#create-a-random-forest-model", + "href": "slides/intro-04-evaluating-models.html#create-a-random-forest-model", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-2", - "href": "slides/intro-03-what-makes-a-model.html#your-turn-2", - "title": "3 - What makes a model?", + "objectID": "slides/intro-04-evaluating-models.html#create-a-random-forest-model-1", + "href": "slides/intro-04-evaluating-models.html#create-a-random-forest-model-1", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + }, + { + "objectID": "slides/intro-04-evaluating-models.html#your-turn-6", + "href": "slides/intro-04-evaluating-models.html#your-turn-6", + "title": "4 - Evaluating models", "section": "Your turn", - "text": "Your turn\n\nRun the tree_wflow chunk in your intro-03-classwork.qmd.\n\ntree_spec <- decision_tree() |> \n set_mode(\"classification\")\n\ntree_wflow <- workflow() |>\n add_formula(forested ~ .) |>\n add_model(tree_spec)\n\ntree_wflow\n\nEdit this code to make a workflow with your own model of choice.\n\nExtension/Challenge: Other than formulas, what kinds of preprocessors are supported?\n\n\n\n−+\n05:00" + "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#predict-with-your-model", - "href": "slides/intro-03-what-makes-a-model.html#predict-with-your-model", - "title": "3 - What makes a model?", - "section": "Predict with your model ", - "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(forested ~ ., tree_spec) |> \n fit(data = forested_train)" + "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance-4", + "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance-4", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nctrl_forested <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, forested_folds, control = ctrl_forested)\ncollect_metrics(rf_res)\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.755 10 0.00482 pre0_mod0_post0\n#> 2 brier_class binary 0.167 10 0.00321 pre0_mod0_post0\n#> 3 roc_auc binary 0.757 10 0.0103 pre0_mod0_post0" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-3", - "href": "slides/intro-03-what-makes-a-model.html#your-turn-3", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = forested_test)\nWhat do you notice about the structure of the result?\n\n\n\n−+\n03:00" + "objectID": "slides/intro-04-evaluating-models.html#the-whole-game---status-update-1", + "href": "slides/intro-04-evaluating-models.html#the-whole-game---status-update-1", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-4", - "href": "slides/intro-03-what-makes-a-model.html#your-turn-4", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = forested_test)\nHow does the output compare to the output from predict()?\n\n\n\n−+\n03:00" + "objectID": "slides/intro-04-evaluating-models.html#the-final-fit", + "href": "slides/intro-04-evaluating-models.html#the-final-fit", + "title": "4 - Evaluating models", + "section": "The final fit ", + "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# forested_split has train + test info\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [8749/2188]> train/test split <tibble> <tibble> <tibble> <workflow>" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#understand-your-model", - "href": "slides/intro-03-what-makes-a-model.html#understand-your-model", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?" + "objectID": "slides/intro-04-evaluating-models.html#what-is-in-final_fit", + "href": "slides/intro-04-evaluating-models.html#what-is-in-final_fit", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.761 pre0_mod0_post0\n#> 2 roc_auc binary 0.761 pre0_mod0_post0\n#> 3 brier_class binary 0.162 pre0_mod0_post0\n\n\nThese are metrics computed with the test set" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#understand-your-model-1", - "href": "slides/intro-03-what-makes-a-model.html#understand-your-model-1", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit |>\n extract_fit_engine() |>\n rpart.plot()\n\nYou can extract_*() several components of your fitted workflow.\n\n⚠️ Never predict() with any extracted components!\n\nroundint = FALSE is only to quiet a warning" + "objectID": "slides/intro-04-evaluating-models.html#what-is-in-final_fit-1", + "href": "slides/intro-04-evaluating-models.html#what-is-in-final_fit-1", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 2,188 × 7\n#> .pred_class .pred_Yes .pred_No id forested .row .config \n#> <fct> <dbl> <dbl> <chr> <fct> <int> <chr> \n#> 1 Yes 0.957 0.0429 train/test split Yes 4 pre0_mod0_pos…\n#> 2 Yes 0.810 0.190 train/test split Yes 8 pre0_mod0_pos…\n#> 3 Yes 0.826 0.174 train/test split Yes 10 pre0_mod0_pos…\n#> 4 Yes 0.842 0.158 train/test split Yes 19 pre0_mod0_pos…\n#> 5 Yes 0.863 0.137 train/test split Yes 23 pre0_mod0_pos…\n#> 6 Yes 0.585 0.415 train/test split No 28 pre0_mod0_pos…\n#> 7 Yes 0.847 0.153 train/test split Yes 34 pre0_mod0_pos…\n#> 8 Yes 0.605 0.395 train/test split No 35 pre0_mod0_pos…\n#> 9 Yes 0.964 0.0362 train/test split Yes 38 pre0_mod0_pos…\n#> 10 Yes 0.926 0.0744 train/test split Yes 40 pre0_mod0_pos…\n#> # ℹ 2,178 more rows" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#understand-your-model-2", - "href": "slides/intro-03-what-makes-a-model.html#understand-your-model-2", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" + "objectID": "slides/intro-04-evaluating-models.html#what-is-in-final_fit-2", + "href": "slides/intro-04-evaluating-models.html#what-is-in-final_fit-2", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 8749 \n#> Number of independent variables: 18 \n#> Mtry: 4 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.1671606\n\n\nUse this for prediction on new data, like for deploying" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-5", - "href": "slides/intro-03-what-makes-a-model.html#your-turn-5", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\n\nExtract the model engine object from your fitted workflow and check it out.\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" + "objectID": "slides/intro-04-evaluating-models.html#the-whole-game", + "href": "slides/intro-04-evaluating-models.html#the-whole-game", + "title": "4 - Evaluating models", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "slides/intro-03-what-makes-a-model.html#the-whole-game---status-update", - "href": "slides/intro-03-what-makes-a-model.html#the-whole-game---status-update", - "title": "3 - What makes a model?", - "section": "The whole game - status update", - "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" + "objectID": "slides/intro-02-data-budget.html#data-on-forests-in-georgia", + "href": "slides/intro-02-data-budget.html#data-on-forests-in-georgia", + "title": "2 - Your data budget", + "section": "Data on forests in Georgia", + "text": "Data on forests in Georgia\n\n\n\nThe U.S. Forest Service maintains ML models to predict whether a plot of land is “forested.”\nThis classification is important for all sorts of research, legislation, and land management purposes.\nPlots are typically remeasured every 10 years and this dataset contains the most recent measurement per plot.\nType ?forested_ga to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/251793/forest-mountain" }, { - "objectID": "slides/intro-01-introduction.html#who-are-you", - "href": "slides/intro-01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have some exposure to basic statistical concepts like linear models and residuals\nYou do not need intermediate or expert familiarity with modeling or ML" + "objectID": "slides/intro-02-data-budget.html#data-on-forests-in-georgia-1", + "href": "slides/intro-02-data-budget.html#data-on-forests-in-georgia-1", + "title": "2 - Your data budget", + "section": "Data on forests in Georgia", + "text": "Data on forests in Georgia\n\n\n\nN = 10937 plots of land, one from each of 10937 6000-acre hexagons in Georgia.\nA nominal outcome, forested, with levels \"Yes\" and \"No\", measured “on-the-ground.”\n18 remotely-sensed and easily-accessible predictors:\n\nnumeric variables based on weather and topography.\nnominal variables based on classifications from other governmental orgs.\n\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/67614/forest\n\n\n\nThose nominal variables are classifications similar to “forested” but from other agencies. e.g. land_type is from the European Space Agency, and is a remotely-sensed 3-class distribution based on predictions for how the land is used. (The data also includes a county variable, which is not itself from a model.)" }, { - "objectID": "slides/intro-01-introduction.html#who-are-tidymodels", - "href": "slides/intro-01-introduction.html#who-are-tidymodels", - "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\n+ our TA today, Kristin Bott!\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "slides/intro-02-data-budget.html#checklist-for-predictors", + "href": "slides/intro-02-data-budget.html#checklist-for-predictors", + "title": "2 - Your data budget", + "section": "Checklist for predictors", + "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?\n\n\n\nre: ethics – what issues might arise from releasing the true lat and lon? In reality, these lat and lon are slightly jittered to help ensure trust with landowners who allow surveyers to come take measurements." }, { - "objectID": "slides/intro-01-introduction.html#section-2", - "href": "slides/intro-01-introduction.html#section-2", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free): TODO-ADD-LATER" + "objectID": "slides/intro-02-data-budget.html#data-on-forests-in-georgia-2", + "href": "slides/intro-02-data-budget.html#data-on-forests-in-georgia-2", + "title": "2 - Your data budget", + "section": "Data on forests in Georgia", + "text": "Data on forests in Georgia\n\nlibrary(tidymodels)\nlibrary(forested)\n\nforested_ga\n#> # A tibble: 10,937 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 2007 14 0 0 No tree 13.9\n#> 2 Yes 2007 66 -53 10 Tree 13.8\n#> 3 Yes 2006 59 -82 6 No tree 13.5\n#> 4 Yes 2007 116 -78 20 Tree 12.3\n#> 5 Yes 2006 283 63 13 Tree 10.0\n#> 6 Yes 2007 250 63 14 Tree 10.8\n#> 7 Yes 2007 58 31 1 No tree 13.8\n#> 8 Yes 2023 140 56 11 Tree 12.2\n#> 9 Yes 2024 118 72 17 Tree 12.2\n#> 10 Yes 2024 217 -46 13 Tree 12.4\n#> # ℹ 10,927 more rows\n#> # ℹ 12 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>, county <fct>" }, { - "objectID": "slides/intro-01-introduction.html#section-3", - "href": "slides/intro-01-introduction.html#section-3", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "slides/intro-02-data-budget.html#data-splitting-and-spending", + "href": "slides/intro-02-data-budget.html#data-splitting-and-spending", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." }, { - "objectID": "slides/intro-01-introduction.html#plan-for-this-workshop", - "href": "slides/intro-01-introduction.html#plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Plan for this workshop", - "text": "Plan for this workshop\n\nYour data budget\nWhat makes a model\nEvaluating models\nTuning models" + "objectID": "slides/intro-02-data-budget.html#data-splitting-and-spending-1", + "href": "slides/intro-02-data-budget.html#data-splitting-and-spending-1", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending" }, { - "objectID": "slides/intro-01-introduction.html#section-4", - "href": "slides/intro-01-introduction.html#section-4", - "title": "1 - Introduction", - "section": "👀", - "text": "👀\n\n\nhttps://workshops.tidymodels.org" + "objectID": "slides/intro-02-data-budget.html#data-splitting-and-spending-2", + "href": "slides/intro-02-data-budget.html#data-splitting-and-spending-2", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." }, { - "objectID": "slides/intro-01-introduction.html#what-is-machine-learning", - "href": "slides/intro-01-introduction.html#what-is-machine-learning", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" + "objectID": "slides/intro-02-data-budget.html#your-turn", + "href": "slides/intro-02-data-budget.html#your-turn", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" }, { - "objectID": "slides/intro-01-introduction.html#what-is-machine-learning-1", - "href": "slides/intro-01-introduction.html#what-is-machine-learning-1", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "slides/intro-02-data-budget.html#the-initial-split", + "href": "slides/intro-02-data-budget.html#the-initial-split", + "title": "2 - Your data budget", + "section": "The initial split ", + "text": "The initial split \n\nset.seed(123)\nforested_split <- initial_split(forested_ga)\nforested_split\n#> <Training/Testing/Total>\n#> <8202/2735/10937>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data" }, { - "objectID": "slides/intro-01-introduction.html#what-is-machine-learning-2024-edition", - "href": "slides/intro-01-introduction.html#what-is-machine-learning-2024-edition", - "title": "1 - Introduction", - "section": "What is machine learning? (2024 edition)", - "text": "What is machine learning? (2024 edition)\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/\n\n\nIn the early 2010s, “Artificial intelligence” (AI) was largely synonymous with what we’ll refer to as “machine learning” in this workshop. In the late 2010s and early 2020s, AI usually referred to deep learning methods. Since the release of ChatGPT in late 2022, “AI” has come to also encompass large language models / generative models." + "objectID": "slides/intro-02-data-budget.html#what-is-set.seed", + "href": "slides/intro-02-data-budget.html#what-is-set.seed", + "title": "2 - Your data budget", + "section": "What is set.seed()?", + "text": "What is set.seed()?\nTo create that split of the data, R generates “pseudo-random” numbers: while they are made to behave like random numbers, their generation is deterministic given a “seed”.\nThis allows us to reproduce results by setting that seed.\nWhich seed you pick doesn’t matter, as long as you don’t try a bunch of seeds and pick the one that gives you the best performance." }, { - "objectID": "slides/intro-01-introduction.html#what-is-machine-learning-2", - "href": "slides/intro-01-introduction.html#what-is-machine-learning-2", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "slides/intro-02-data-budget.html#accessing-the-data", + "href": "slides/intro-02-data-budget.html#accessing-the-data", + "title": "2 - Your data budget", + "section": "Accessing the data ", + "text": "Accessing the data \n\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)" }, { - "objectID": "slides/intro-01-introduction.html#your-turn", - "href": "slides/intro-01-introduction.html#your-turn", - "title": "1 - Introduction", - "section": "Your turn", - "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + "objectID": "slides/intro-02-data-budget.html#the-training-set", + "href": "slides/intro-02-data-budget.html#the-training-set", + "title": "2 - Your data budget", + "section": "The training set", + "text": "The training set\n\nforested_train\n#> # A tibble: 8,202 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 1997 66 82 10 Tree 12.2 \n#> 2 No 1997 284 -99 58 Tree 10.3 \n#> 3 Yes 2022 130 86 15 Tree 11.8 \n#> 4 Yes 2021 202 -55 3 Tree 10.7 \n#> 5 Yes 1995 75 -89 1 Tree 13.8 \n#> 6 No 1995 110 -53 5 Tree 12.4 \n#> 7 Yes 2022 111 73 12 Tree 11.5 \n#> 8 Yes 1997 230 96 14 Tree 9.98\n#> 9 Yes 2002 160 -88 13 Tree 11.1 \n#> 10 Yes 2020 39 9 6 Tree 13.9 \n#> # ℹ 8,192 more rows\n#> # ℹ 12 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>, county <fct>" }, { - "objectID": "slides/intro-01-introduction.html#what-is-tidymodels", - "href": "slides/intro-01-introduction.html#what-is-tidymodels", - "title": "1 - Introduction", - "section": "What is tidymodels? ", - "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.3.0 ──\n#> ✔ broom 1.0.9 ✔ rsample 1.3.1\n#> ✔ dials 1.4.2 ✔ tibble 3.3.0\n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1\n#> ✔ infer 1.0.9 ✔ tune 2.0.0\n#> ✔ modeldata 1.5.1 ✔ workflows 1.3.0\n#> ✔ parsnip 1.3.3 ✔ workflowsets 1.1.1\n#> ✔ purrr 1.1.0 ✔ yardstick 1.3.2\n#> ✔ recipes 1.3.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()" + "objectID": "slides/intro-02-data-budget.html#the-training-set-1", + "href": "slides/intro-02-data-budget.html#the-training-set-1", + "title": "2 - Your data budget", + "section": "The training set", + "text": "The training set\n\nforested_train |>\n select(where(is.factor))\n#> # A tibble: 8,202 × 4\n#> forested tree_no_tree land_type county \n#> <fct> <fct> <fct> <fct> \n#> 1 Yes Tree Tree Muscogee \n#> 2 No Tree Tree Polk \n#> 3 Yes Tree Tree Hancock \n#> 4 Yes Tree Tree Oglethorpe\n#> 5 Yes Tree Tree Berrien \n#> 6 No Tree Non-tree vegetation Dooly \n#> 7 Yes Tree Tree Columbia \n#> 8 Yes Tree Tree Walker \n#> 9 Yes Tree Tree Greene \n#> 10 Yes Tree Tree Wayne \n#> # ℹ 8,192 more rows\n\n\n\nThe tree_no_tree and land_type variables are both outputs from other machine learning models. Companies make their own classifications of forest vs. non-forest based on satellite images and then sell them. Those classifications are available at prediction time; the FIA will happily spend the few thousand dollars to buy the data for the whole country." }, { - "objectID": "slides/intro-01-introduction.html#the-whole-game", - "href": "slides/intro-01-introduction.html#the-whole-game", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\nRoadmap for today\nMinimal version of predictive modeling process\nFeature engineering and tuning as iterative extensions" + "objectID": "slides/intro-02-data-budget.html#the-test-set", + "href": "slides/intro-02-data-budget.html#the-test-set", + "title": "2 - Your data budget", + "section": "The test set ", + "text": "The test set \n🙈\n\nThere are 2735 rows and 19 columns in the test set." }, { - "objectID": "slides/intro-01-introduction.html#the-whole-game-1", - "href": "slides/intro-01-introduction.html#the-whole-game-1", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/intro-02-data-budget.html#your-turn-1", + "href": "slides/intro-02-data-budget.html#your-turn-1", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\nWe recommend using the .qmd files in the classwork/ folder for code exercises. They set you up with the code from the slides.\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-01-introduction.html#the-whole-game-2", - "href": "slides/intro-01-introduction.html#the-whole-game-2", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." + "objectID": "slides/intro-02-data-budget.html#data-splitting-and-spending-3", + "href": "slides/intro-02-data-budget.html#data-splitting-and-spending-3", + "title": "2 - Your data budget", + "section": "Data splitting and spending ", + "text": "Data splitting and spending \n\nset.seed(123)\nforested_split <- initial_split(forested_ga, prop = 0.8)\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)\n\nnrow(forested_train)\n#> [1] 8749\nnrow(forested_test)\n#> [1] 2188" }, { - "objectID": "slides/intro-01-introduction.html#the-whole-game-3", - "href": "slides/intro-01-introduction.html#the-whole-game-3", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/intro-02-data-budget.html#your-turn-2", + "href": "slides/intro-02-data-budget.html#your-turn-2", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nExplore the forested_train data on your own!\n\nWhat’s the distribution of the outcome, forested?\nWhat’s the distribution of numeric variables like precip_annual?\nHow does the distribution of forested differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" }, { - "objectID": "slides/intro-01-introduction.html#the-whole-game-4", - "href": "slides/intro-01-introduction.html#the-whole-game-4", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/intro-02-data-budget.html#section-1", + "href": "slides/intro-02-data-budget.html#section-1", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = forested)) +\n geom_bar()" }, { - "objectID": "slides/intro-01-introduction.html#the-whole-game-5", - "href": "slides/intro-01-introduction.html#the-whole-game-5", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/intro-02-data-budget.html#section-2", + "href": "slides/intro-02-data-budget.html#section-2", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = forested, fill = tree_no_tree)) +\n geom_bar()" }, { - "objectID": "slides/intro-01-introduction.html#the-whole-game-6", - "href": "slides/intro-01-introduction.html#the-whole-game-6", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/intro-02-data-budget.html#section-3", + "href": "slides/intro-02-data-budget.html#section-3", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"identity\", alpha = .7)" }, { - "objectID": "slides/intro-01-introduction.html#the-whole-game-7", - "href": "slides/intro-01-introduction.html#the-whole-game-7", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/intro-02-data-budget.html#section-4", + "href": "slides/intro-02-data-budget.html#section-4", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"fill\")" }, { - "objectID": "slides/intro-01-introduction.html#lets-install-some-packages", - "href": "slides/intro-01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"lightgbm\", \"lme4\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)" + "objectID": "slides/intro-02-data-budget.html#section-5", + "href": "slides/intro-02-data-budget.html#section-5", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = lon, y = lat, col = forested)) +\n geom_point()" }, { - "objectID": "slides/intro-01-introduction.html#our-versions", - "href": "slides/intro-01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nR version 4.5.1 (2025-06-13), Quarto (1.7.32)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.4.0\n\n\nbroom\n1.0.9\n\n\nCubist\n0.5.0\n\n\ndials\n1.4.2\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.4\n\n\nembed\n1.1.5\n\n\nfinetune\n1.2.1\n\n\nforested\n0.2.0\n\n\nFormula\n1.2-5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nggplot2\n3.5.2\n\n\nlattice\n0.22-7\n\n\nlightgbm\n4.6.0\n\n\nlme4\n1.1-37\n\n\nmodeldata\n1.5.1\n\n\nparallelly\n1.45.1\n\n\nparsnip\n1.3.3\n\n\nplotmo\n3.6.4\n\n\nplotrix\n3.8-4\n\n\nplumber\n1.3.0\n\n\nprobably\n1.1.1\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\npurrr\n1.1.0\n\n\nranger\n0.17.0\n\n\nrecipes\n1.3.1\n\n\nrpart\n4.1.24\n\n\nrpart.plot\n3.1.3\n\n\nrsample\n1.3.1\n\n\nrules\n1.0.2\n\n\nscales\n1.4.0\n\n\nsplines2\n0.5.4\n\n\nstacks\n1.1.1\n\n\ntext2vec\n0.6.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntextrecipes\n1.1.0\n\n\ntibble\n3.3.0\n\n\ntidymodels\n1.3.0\n\n\ntidyr\n1.3.1\n\n\ntune\n2.0.0\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.3.0\n\n\nworkflowsets\n1.1.1\n\n\nyardstick\n1.3.2" + "objectID": "slides/intro-02-data-budget.html#the-whole-game---status-update", + "href": "slides/intro-02-data-budget.html#the-whole-game---status-update", + "title": "2 - Your data budget", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { "objectID": "slides/extras-transit-case-study.html#chicago-l-train-data", @@ -7923,6 +7279,13 @@ "section": "What is set.seed()?", "text": "What is set.seed()?\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" }, + { + "objectID": "slides/annotations.html#understand-your-model", + "href": "slides/annotations.html#understand-your-model", + "title": "Annotations", + "section": "Understand your model", + "text": "Understand your model\nWe’ve abbreviated the code necessary to make the exact plot on the previous slide. The full code is:\nlibrary(rpart.plot)\nsplit_fun <- function(x, labs, digits, varlen, faclen) {\n for (i in 1:length(labs)) {\n if (grepl(\",\", labs[i])) {\n parts <- strsplit(labs[i], \",\")[[1]]\n parts <- trimws(parts)\n if (length(parts) > 5) {\n parts <- c(parts[1:5], \"...\")\n }\n labs[i] <- paste(parts, collapse = \", \")\n }\n labs[i] <- paste(strwrap(labs[i], width = 15), collapse = \"\\n\")\n }\n labs\n}\n\ntree_fit |>\n extract_fit_engine() |>\n rpart.plot(\n roundint = FALSE,\n type = 3,\n clip.right.labs = FALSE,\n split.fun = split_fun\n )" + }, { "objectID": "slides/annotations.html#what-is-wrong-with-this", "href": "slides/annotations.html#what-is-wrong-with-this", @@ -7966,11 +7329,25 @@ "text": "Different types of grids\nMore on space-filling designs in Chapters 4 and 5 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences.\nThese designs also scale well as the number of tuning parameters grows. They are designed to fill the predictor space efficiently." }, { - "objectID": "slides/annotations.html#update-parameter-ranges", - "href": "slides/annotations.html#update-parameter-ranges", + "objectID": "slides/annotations.html#extract-and-update-parameters", + "href": "slides/annotations.html#extract-and-update-parameters", "title": "Annotations", - "section": "Update parameter ranges", - "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object |> \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) |> \n step_spline_natural(dis, deg_free = tune()) |> \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" + "section": "Extract and update parameters", + "text": "Extract and update parameters\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object |> \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) |> \n step_spline_natural(dis, deg_free = tune()) |> \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" + }, + { + "objectID": "slides/annotations.html#grid-search", + "href": "slides/annotations.html#grid-search", + "title": "Annotations", + "section": "Grid Search", + "text": "Grid Search\nThere might be some warnings that\n\n“Early stopping occurred at epoch 3 due to numerical overflow of the loss function.”\n\ntorch can be very aggressive about moving in the direction of the gradient. In some cases, it moves to a space where the gradient is not well-behaved (e.g., flat in multiple directions, saddle point, etc).\nThis can cause abnormally large parameter values with magnitudes larger than double precision variables can hold.\nIn this case, brulee stops the optimization and returns the parameters from the last best iteration." + }, + { + "objectID": "slides/annotations.html#running-in-parallel", + "href": "slides/annotations.html#running-in-parallel", + "title": "Annotations", + "section": "Running in parallel", + "text": "Running in parallel\nWe usually leave one or two cores available to work with when we run in parallel.\nIf you are using an HPC system, be careful to use only the cores allocated to you.\nAlso, memory is duplicated for each worker. If your main R session is using 1GB of memory, using 10 workers requires 11GB.\nComparing these methods:\n\nThey usually perform about the same.\nfuture is more mature and is geared towards users and developers.\nmirai contains its own queuing system and can be more efficient in allocating work." }, { "objectID": "slides/annotations.html#early-stopping-for-boosted-trees", @@ -7993,6 +7370,13 @@ "section": "Acquisition Functions", "text": "Acquisition Functions\nMore references:\n\nChapter 7 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences\nBayesian Optimization, Chapter 6 (pdf)\nGaussian Processes for Machine Learning" }, + { + "objectID": "slides/annotations.html#isomap-with-recipes", + "href": "slides/annotations.html#isomap-with-recipes", + "title": "Annotations", + "section": "isomap with recipes", + "text": "isomap with recipes\nThe results from isomap unlike UMAP is not able to be distorted in the same way. Here the number of nieghbors change between the charts. Each of the charts are an application of isomap on the same completely random data. The points are in different locations but none of the charts appear to show any patterns.\n\n#> Warning: The `x` argument of `as_tibble.matrix()` must have unique column\n#> names if `.name_repair` is omitted as of tibble 2.0.0.\n#> ℹ Using compatibility `.name_repair`.\n#> 2025-09-11 19:15:59.990984: Isomap START\n#> 2025-09-11 19:15:59.991269: constructing knn graph\n#> 2025-09-11 19:16:00.002963: calculating geodesic distances\n#> 2025-09-11 19:16:00.004455: Classical Scaling\n#> 2025-09-11 19:16:00.107195: Isomap START\n#> 2025-09-11 19:16:00.107374: constructing knn graph\n#> 2025-09-11 19:16:00.115635: calculating geodesic distances\n#> 2025-09-11 19:16:00.117154: Classical Scaling\n#> 2025-09-11 19:16:00.211276: Isomap START\n#> 2025-09-11 19:16:00.211468: constructing knn graph\n#> 2025-09-11 19:16:00.220394: calculating geodesic distances\n#> 2025-09-11 19:16:00.222549: Classical Scaling\n#> 2025-09-11 19:16:00.321161: Isomap START\n#> 2025-09-11 19:16:00.321347: constructing knn graph\n#> 2025-09-11 19:16:00.330446: calculating geodesic distances\n#> 2025-09-11 19:16:00.333601: Classical Scaling" + }, { "objectID": "slides/annotations.html#per-agent-statistics", "href": "slides/annotations.html#per-agent-statistics", @@ -8015,12540 +7399,12470 @@ "text": "A recipe - handle correlations\nIn this code chunk, What’s the story with !!stations?\n\nchi_pca_rec <- \n chi_rec |> \n step_normalize(all_of(!!stations)) |> \n step_pca(all_of(!!stations), num_comp = tune())\n\nstations is a vector of names of 20 columns that we want to use in the steps. If the list were shorter, we could type them in (e.g., c(\"col1\", \"col2\") etc.).\nThe vector lives in our global workspace, and if we are in parallel, the worker processes might not have access to stations. The !! (frequently said as “bang bang”) inserts the actual contents of the vector into the all_of() calls so that it looks like you just typed it in.\nThis means that the parallel process workers have a copy of the data in their reach and the code will run without error." }, { - "objectID": "slides/advanced-05-iterative.html#previously---setup", - "href": "slides/advanced-05-iterative.html#previously---setup", - "title": "5 - Iterative Search", - "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" - }, - { - "objectID": "slides/advanced-05-iterative.html#previously---data-usage", - "href": "slides/advanced-05-iterative.html#previously---data-usage", - "title": "5 - Iterative Search", - "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" - }, - { - "objectID": "slides/advanced-05-iterative.html#our-boosting-model", - "href": "slides/advanced-05-iterative.html#our-boosting-model", - "title": "5 - Iterative Search", - "section": "Our Boosting Model", - "text": "Our Boosting Model\nWe used feature hashing to generate a smaller set of indicator columns to deal with the large number of levels for the agent and country predictors.\n\nTree-based models (and a few others) don’t require indicators for categorical predictors. They can split on these variables as-is.\n\nWe’ll keep all categorical predictors as factors and focus on optimizing additional boosting parameters." - }, - { - "objectID": "slides/advanced-05-iterative.html#our-boosting-model-1", - "href": "slides/advanced-05-iterative.html#our-boosting-model-1", - "title": "5 - Iterative Search", - "section": "Our Boosting Model", - "text": "Our Boosting Model\n\nlgbm_spec <- \n boost_tree(trees = 1000, learn_rate = tune(), min_n = tune(), \n tree_depth = tune(), loss_reduction = tune(), \n stop_iter = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(avg_price_per_room ~ ., lgbm_spec)\n\nlgbm_param <- \n lgbm_wflow |>\n extract_parameter_set_dials() |>\n update(learn_rate = learn_rate(c(-5, -1)))" - }, - { - "objectID": "slides/advanced-05-iterative.html#iterative-search", - "href": "slides/advanced-05-iterative.html#iterative-search", - "title": "5 - Iterative Search", - "section": "Iterative Search", - "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our boosted tree.\n\nWe could do something like:\nmae_pred <- lm(mae ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." - }, - { - "objectID": "slides/advanced-05-iterative.html#iterative-search-1", - "href": "slides/advanced-05-iterative.html#iterative-search-1", - "title": "5 - Iterative Search", - "section": "Iterative Search", - "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between MAE and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" - }, - { - "objectID": "slides/advanced-05-iterative.html#a-large-grid", - "href": "slides/advanced-05-iterative.html#a-large-grid", - "title": "5 - Iterative Search", - "section": "A Large Grid", - "text": "A Large Grid" - }, - { - "objectID": "slides/advanced-05-iterative.html#a-three-point-grid", - "href": "slides/advanced-05-iterative.html#a-three-point-grid", - "title": "5 - Iterative Search", - "section": "A Three Point Grid", - "text": "A Three Point Grid" - }, - { - "objectID": "slides/advanced-05-iterative.html#gaussian-processes-and-optimization", - "href": "slides/advanced-05-iterative.html#gaussian-processes-and-optimization", - "title": "5 - Iterative Search", - "section": "Gaussian Processes and Optimization", - "text": "Gaussian Processes and Optimization\nWe can make a “meta-model” with a small set of historical performance results.\nGaussian Processes (GP) models are a good choice to model performance.\n\nIt is a Bayesian model so we are using Bayesian Optimization (BO).\nFor regression, we can assume that our data are multivariate normal.\nWe also define a covariance function for the variance relationship between data points. A common one is:\n\n\\[\\operatorname{cov}(\\boldsymbol{x}_i, \\boldsymbol{x}_j) = \\exp\\left(-\\frac{1}{2}|\\boldsymbol{x}_i - \\boldsymbol{x}_j|^2\\right) + \\sigma^2_{ij}\\]\n\nGPs are good because\n\nthey are flexible regression models (in the sense that splines are flexible).\nwe need to get mean and variance predictions (and they are Bayesian)\ntheir variability is based on spatial distances.\n\nSome people use random forests (with conformal variance estimates) or other methods but GPs are most popular." - }, - { - "objectID": "slides/advanced-05-iterative.html#predicting-candidates", - "href": "slides/advanced-05-iterative.html#predicting-candidates", - "title": "5 - Iterative Search", - "section": "Predicting Candidates", - "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. MAE)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." - }, - { - "objectID": "slides/advanced-05-iterative.html#your-turn", - "href": "slides/advanced-05-iterative.html#your-turn", - "title": "5 - Iterative Search", - "section": "Your turn", - "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize MAE.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" - }, - { - "objectID": "slides/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", - "href": "slides/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", - "title": "5 - Iterative Search", - "section": "GP Fit (ribbon is mean +/- 1SD)", - "text": "GP Fit (ribbon is mean +/- 1SD)" - }, - { - "objectID": "slides/advanced-05-iterative.html#choosing-new-candidates", - "href": "slides/advanced-05-iterative.html#choosing-new-candidates", - "title": "5 - Iterative Search", - "section": "Choosing New Candidates", - "text": "Choosing New Candidates\nThis isn’t a very good fit but we can still use it.\nHow can we use the outputs to choose the next point to measure?\n\nAcquisition functions take the predicted mean and variance and use them to balance:\n\nexploration: new candidates should explore new areas.\nexploitation: new candidates must stay near existing values.\n\nExploration focuses on the variance, exploitation is about the mean." - }, - { - "objectID": "slides/advanced-05-iterative.html#acquisition-functions", - "href": "slides/advanced-05-iterative.html#acquisition-functions", - "title": "5 - Iterative Search", - "section": "Acquisition Functions", - "text": "Acquisition Functions\nWe’ll use an acquisition function to select a new candidate.\nThe most popular method appears to be expected improvement (EI) above the current best results.\n\nZero at existing data points.\nThe expected improvement is integrated over all possible improvement (“expected” in the probability sense).\n\nWe would probably pick the point with the largest EI as the next point.\n(There are other functions beyond EI.)" - }, - { - "objectID": "slides/advanced-05-iterative.html#expected-improvement", - "href": "slides/advanced-05-iterative.html#expected-improvement", - "title": "5 - Iterative Search", - "section": "Expected Improvement", - "text": "Expected Improvement" - }, - { - "objectID": "slides/advanced-05-iterative.html#iteration", - "href": "slides/advanced-05-iterative.html#iteration", - "title": "5 - Iterative Search", - "section": "Iteration", - "text": "Iteration\nOnce we pick the candidate point, we measure performance for it (e.g. resampling).\n\nAnother GP is fit, EI is recomputed, and so on.\n\nWe stop when we have completed the allowed number of iterations or if we don’t see any improvement after a pre-set number of attempts." - }, - { - "objectID": "slides/advanced-05-iterative.html#gp-fit-with-four-points", - "href": "slides/advanced-05-iterative.html#gp-fit-with-four-points", - "title": "5 - Iterative Search", - "section": "GP Fit with four points", - "text": "GP Fit with four points" - }, - { - "objectID": "slides/advanced-05-iterative.html#expected-improvement-1", - "href": "slides/advanced-05-iterative.html#expected-improvement-1", - "title": "5 - Iterative Search", - "section": "Expected Improvement", - "text": "Expected Improvement" - }, - { - "objectID": "slides/advanced-05-iterative.html#gp-evolution", - "href": "slides/advanced-05-iterative.html#gp-evolution", - "title": "5 - Iterative Search", - "section": "GP Evolution", - "text": "GP Evolution" - }, - { - "objectID": "slides/advanced-05-iterative.html#expected-improvement-evolution", - "href": "slides/advanced-05-iterative.html#expected-improvement-evolution", - "title": "5 - Iterative Search", - "section": "Expected Improvement Evolution", - "text": "Expected Improvement Evolution" - }, - { - "objectID": "slides/advanced-05-iterative.html#bo-in-tidymodels", - "href": "slides/advanced-05-iterative.html#bo-in-tidymodels", - "title": "5 - Iterative Search", - "section": "BO in tidymodels", - "text": "BO in tidymodels\nWe’ll use a function called tune_bayes() that has very similar syntax to tune_grid().\n\nIt has an additional initial argument for the initial set of performance estimates and parameter combinations for the GP model." - }, - { - "objectID": "slides/advanced-05-iterative.html#initial-grid-points", - "href": "slides/advanced-05-iterative.html#initial-grid-points", - "title": "5 - Iterative Search", - "section": "Initial grid points", - "text": "Initial grid points\ninitial can be the results of another tune_*() function or an integer (in which case tune_grid() is used under to hood to make such an initial set of results).\n\nWe’ll run the optimization more than once, so let’s make an initial grid of results to serve as the substrate for the BO.\nI suggest at least the number of tuning parameters plus two as the initial grid for BO." - }, - { - "objectID": "slides/advanced-05-iterative.html#an-initial-grid", - "href": "slides/advanced-05-iterative.html#an-initial-grid", - "title": "5 - Iterative Search", - "section": "An Initial Grid", - "text": "An Initial Grid\n\nreg_metrics <- metric_set(mae, rsq)\n\nset.seed(12)\ninit_res <-\n lgbm_wflow |>\n tune_grid(\n resamples = hotel_rs,\n grid = nrow(lgbm_param) + 2,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(init_res, metric = \"mae\") |> select(-.metric, -.estimator)\n#> # A tibble: 5 × 9\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config \n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> \n#> 1 33 12 0.0215 8.25e- 9 5 10.1 10 0.182 pre0_mod6_post0\n#> 2 14 10 0.1 3.16e+ 1 8 10.2 10 0.192 pre0_mod3_post0\n#> 3 8 3 0.00464 1 e-10 11 14.4 10 0.305 pre0_mod2_post0\n#> 4 40 1 0.001 4.64e- 3 14 36.1 10 0.408 pre0_mod7_post0\n#> 5 27 15 0.000215 6.81e- 7 20 44.5 10 0.371 pre0_mod5_post0" - }, - { - "objectID": "slides/advanced-05-iterative.html#bo-using-tidymodels", - "href": "slides/advanced-05-iterative.html#bo-using-tidymodels", - "title": "5 - Iterative Search", - "section": "BO using tidymodels", - "text": "BO using tidymodels\n\nctrl_bo <- control_bayes(verbose_iter = TRUE) # <- for demonstration\n\nset.seed(15)\nlgbm_bayes_res <-\n lgbm_wflow |>\n tune_bayes(\n resamples = hotel_rs,\n initial = init_res, # <- initial results\n iter = 20,\n param_info = lgbm_param,\n control = ctrl_bo,\n metrics = reg_metrics\n )\n#> Optimizing mae using the expected improvement\n#> \n#> ── Iteration 1 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=10.05 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=6, learn_rate=0.0857, loss_reduction=2.6e-10, stop_iter=14\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.716 (+/-0.179)\n#> \n#> ── Iteration 2 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.716 (@iter 1)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=14, learn_rate=0.0301, loss_reduction=0.000104, stop_iter=3\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.781 (+/-0.183)\n#> \n#> ── Iteration 3 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.716 (@iter 1)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=12, tree_depth=8, learn_rate=0.0545, loss_reduction=7.34e-08, stop_iter=18\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.787 (+/-0.193)\n#> \n#> ── Iteration 4 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.716 (@iter 1)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=20, tree_depth=5, learn_rate=0.0119, loss_reduction=6.21e-06, stop_iter=10\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.58 (+/-0.208)\n#> \n#> ── Iteration 5 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.716 (@iter 1)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=11, tree_depth=7, learn_rate=0.0713, loss_reduction=6.31, stop_iter=5\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.901 (+/-0.161)\n#> \n#> ── Iteration 6 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.716 (@iter 1)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=6, tree_depth=9, learn_rate=0.0401, loss_reduction=2.37e-10, stop_iter=10\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.666 (+/-0.152)\n#> \n#> ── Iteration 7 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.666 (@iter 6)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=15, tree_depth=4, learn_rate=0.0346, loss_reduction=2.98e-10, stop_iter=10\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.28 (+/-0.155)\n#> \n#> ── Iteration 8 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.666 (@iter 6)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=11, learn_rate=0.0193, loss_reduction=9, stop_iter=19\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.988 (+/-0.188)\n#> \n#> ── Iteration 9 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.666 (@iter 6)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=11, tree_depth=15, learn_rate=0.0797, loss_reduction=1.49e-10, stop_iter=14\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.654 (+/-0.164)\n#> \n#> ── Iteration 10 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.654 (@iter 9)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=11, tree_depth=14, learn_rate=0.044, loss_reduction=0.0078, stop_iter=6\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.767 (+/-0.159)\n#> \n#> ── Iteration 11 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.654 (@iter 9)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=3, tree_depth=14, learn_rate=0.0543, loss_reduction=3.17e-10, stop_iter=18\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.6 (+/-0.149)\n#> \n#> ── Iteration 12 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.6 (@iter 11)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=13, learn_rate=0.0254, loss_reduction=1.64e-05, stop_iter=9\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.834 (+/-0.17)\n#> \n#> ── Iteration 13 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.6 (@iter 11)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=5, tree_depth=14, learn_rate=0.0629, loss_reduction=2.41e-09, stop_iter=10\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.609 (+/-0.168)\n#> \n#> ── Iteration 14 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.6 (@iter 11)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=7, tree_depth=14, learn_rate=0.0592, loss_reduction=6.82e-08, stop_iter=6\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.653 (+/-0.162)\n#> \n#> ── Iteration 15 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.6 (@iter 11)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=3, tree_depth=15, learn_rate=0.049, loss_reduction=2.25e-06, stop_iter=8\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.634 (+/-0.209)\n#> \n#> ── Iteration 16 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.6 (@iter 11)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=4, tree_depth=15, learn_rate=0.0391, loss_reduction=8.31e-08, stop_iter=10\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.673 (+/-0.154)\n#> \n#> ── Iteration 17 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.6 (@iter 11)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=15, learn_rate=0.0544, loss_reduction=2.98e-09, stop_iter=3\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.634 (+/-0.161)\n#> \n#> ── Iteration 18 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.6 (@iter 11)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=3, tree_depth=12, learn_rate=0.0416, loss_reduction=5.12e-09, stop_iter=13\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.66 (+/-0.164)\n#> \n#> ── Iteration 19 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.6 (@iter 11)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=10, learn_rate=0.0467, loss_reduction=1.11e-09, stop_iter=8\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.648 (+/-0.138)\n#> \n#> ── Iteration 20 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.6 (@iter 11)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=3, tree_depth=15, learn_rate=0.0603, loss_reduction=1.06e-10, stop_iter=16\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.638 (+/-0.167)" - }, - { - "objectID": "slides/advanced-05-iterative.html#best-results", - "href": "slides/advanced-05-iterative.html#best-results", - "title": "5 - Iterative Search", - "section": "Best results", - "text": "Best results\n\nshow_best(lgbm_bayes_res, metric = \"mae\") |> select(-.metric, -.estimator)\n#> # A tibble: 5 × 10\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config .iter\n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> <int>\n#> 1 3 14 0.0543 3.17e-10 18 9.60 10 0.149 iter11 11\n#> 2 5 14 0.0629 2.41e- 9 10 9.61 10 0.168 iter13 13\n#> 3 3 15 0.0490 2.25e- 6 8 9.63 10 0.209 iter15 15\n#> 4 2 15 0.0544 2.98e- 9 3 9.63 10 0.161 iter17 17\n#> 5 3 15 0.0603 1.06e-10 16 9.64 10 0.167 iter20 20" - }, - { - "objectID": "slides/advanced-05-iterative.html#plotting-bo-results", - "href": "slides/advanced-05-iterative.html#plotting-bo-results", - "title": "5 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\")" - }, - { - "objectID": "slides/advanced-05-iterative.html#plotting-bo-results-1", - "href": "slides/advanced-05-iterative.html#plotting-bo-results-1", - "title": "5 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"parameters\")" - }, - { - "objectID": "slides/advanced-05-iterative.html#plotting-bo-results-2", - "href": "slides/advanced-05-iterative.html#plotting-bo-results-2", - "title": "5 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\")" - }, - { - "objectID": "slides/advanced-05-iterative.html#enhance", - "href": "slides/advanced-05-iterative.html#enhance", - "title": "5 - Iterative Search", - "section": "ENHANCE", - "text": "ENHANCE\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\") +\n ylim(c(9, 14))" - }, - { - "objectID": "slides/advanced-05-iterative.html#your-turn-1", - "href": "slides/advanced-05-iterative.html#your-turn-1", - "title": "5 - Iterative Search", - "section": "Your turn", - "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\nBonus points: Before the optimization is done, press <esc> and see what happens.\n\n\n\n−+\n10:00" - }, - { - "objectID": "slides/advanced-05-iterative.html#notes", - "href": "slides/advanced-05-iterative.html#notes", - "title": "5 - Iterative Search", - "section": "Notes", - "text": "Notes\n\nStopping tune_bayes() will return the current results.\nParallel processing can still be used to more efficiently measure each candidate point.\nThere are a lot of other iterative methods that you can use.\nThe finetune package also has functions for simulated annealing search." - }, - { - "objectID": "slides/advanced-05-iterative.html#finalizing-the-model", - "href": "slides/advanced-05-iterative.html#finalizing-the-model", - "title": "5 - Iterative Search", - "section": "Finalizing the Model", - "text": "Finalizing the Model\nLet’s say that we’ve tried a lot of different models and we like our lightgbm model the most.\nWhat do we do now?\n\nFinalize the workflow by choosing the values for the tuning parameters.\nFit the model on the entire training set.\nVerify performance using the test set.\nDocument and publish the model(?)" - }, - { - "objectID": "slides/advanced-05-iterative.html#locking-down-the-tuning-parameters", - "href": "slides/advanced-05-iterative.html#locking-down-the-tuning-parameters", - "title": "5 - Iterative Search", - "section": "Locking Down the Tuning Parameters", - "text": "Locking Down the Tuning Parameters\nWe can take the results of the Bayesian optimization and accept the best results:\n\nbest_param <- select_best(lgbm_bayes_res, metric = \"mae\")\nfinal_wflow <- \n lgbm_wflow |> \n finalize_workflow(best_param)\nfinal_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: boost_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> avg_price_per_room ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Boosted Tree Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> min_n = 3\n#> tree_depth = 14\n#> learn_rate = 0.0542665755419299\n#> loss_reduction = 3.17296158491419e-10\n#> stop_iter = 18\n#> \n#> Engine-Specific Arguments:\n#> num_threads = 1\n#> \n#> Computational engine: lightgbm" + "objectID": "slides/advanced-07-feature-selection.html#startup", + "href": "slides/advanced-07-feature-selection.html#startup", + "title": "7 - Feature selection", + "section": "Startup! ", + "text": "Startup! \n\nlibrary(tidymodels)\nlibrary(important)\nlibrary(probably)\nlibrary(mirai)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)\ndaemons(parallel::detectCores())" }, { - "objectID": "slides/advanced-05-iterative.html#the-final-fit", - "href": "slides/advanced-05-iterative.html#the-final-fit", - "title": "5 - Iterative Search", - "section": "The Final Fit", - "text": "The Final Fit\nWe can use individual functions:\nfinal_fit <- final_wflow |> fit(data = hotel_train)\n\n# then predict() or augment() \n# then compute metrics\n\nRemember that there is also a convenience function to do all of this:\n\nset.seed(3893)\nfinal_res <- final_wflow |> last_fit(hotel_split, metrics = reg_metrics)\nfinal_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [3749/1251]> train/test split <tibble [2 × 4]> <tibble [0 × 4]> <tibble [1,251 × 4]> <workflow>" - }, - { - "objectID": "slides/advanced-05-iterative.html#test-set-results", - "href": "slides/advanced-05-iterative.html#test-set-results", - "title": "5 - Iterative Search", - "section": "Test Set Results", - "text": "Test Set Results\n\n\n\nfinal_res |> \n collect_predictions() |> \n cal_plot_regression(\n truth = avg_price_per_room, \n estimate = .pred)\n\nTest set performance:\n\nfinal_res |> collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 9.71 pre0_mod0_post0\n#> 2 rsq standard 0.947 pre0_mod0_post0\n\n\n\n\n\n\n\n\n\n\n\n\nRecall that resampling predicted the MAE to be 9.6." + "objectID": "slides/advanced-07-feature-selection.html#more-startup", + "href": "slides/advanced-07-feature-selection.html#more-startup", + "title": "7 - Feature selection", + "section": "More startup! ", + "text": "More startup! \n\n# Load our example data for this section\n\"https://raw.githubusercontent.com/tidymodels/\" |> \n paste0(\"workshops/main/slides/class_data.RData\") |> \n url() |> \n load()\n\nset.seed(429)\nsim_split <- initial_split(class_data, prop = 0.75, strata = class)\nsim_train <- training(sim_split)\nsim_test <- testing(sim_split)\n\nset.seed(523)\nsim_rs <- vfold_cv(sim_train, v = 10, strata = class)" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#previously---setup", - "href": "slides/advanced-03-tuning-hyperparameters.html#previously---setup", - "title": "3 - Tuning Hyperparameters", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/advanced-07-feature-selection.html#models-and-feature-selection", + "href": "slides/advanced-07-feature-selection.html#models-and-feature-selection", + "title": "7 - Feature selection", + "section": "Models and feature selection", + "text": "Models and feature selection\nSome models automatically remove predictors by never using them in the model:\n\ntree- and rule-based models\nsome regularized models (e.g., glmnet)\nmultivariate adaptive regression splines (MARS)\nRuleFit\nnot really ensembles though\n\nSometimes using irrelevant predictors hurts model performance." }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#previously---data-usage", - "href": "slides/advanced-03-tuning-hyperparameters.html#previously---data-usage", - "title": "3 - Tuning Hyperparameters", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "slides/advanced-07-feature-selection.html#effects-of-extra-predictors", + "href": "slides/advanced-07-feature-selection.html#effects-of-extra-predictors", + "title": "7 - Feature selection", + "section": "Effects of extra predictors", + "text": "Effects of extra predictors\n\n\nAML4TD" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", - "href": "slides/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", - "title": "3 - Tuning Hyperparameters", - "section": "Previously - Feature engineering ", - "text": "Previously - Feature engineering \n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) |>\n step_dummy_hash(company) |>\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors())" + "objectID": "slides/advanced-07-feature-selection.html#general-selection-methods", + "href": "slides/advanced-07-feature-selection.html#general-selection-methods", + "title": "7 - Feature selection", + "section": "General selection methods", + "text": "General selection methods\n\nwrappers: a sequential algorithm proposes feature subsets, fits the model with these subsets, and then determines a better subset from the results.\nfilters: screen predictors before adding them to the model.\n\ntidymodels doesn’t have any wrappers (but see the caret documentation for them)\n\nThe new important package does have filters via recipes." }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#tuning-parameters", - "href": "slides/advanced-03-tuning-hyperparameters.html#tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "slides/advanced-07-feature-selection.html#be-careful", + "href": "slides/advanced-07-feature-selection.html#be-careful", + "title": "7 - Feature selection", + "section": "Be careful!!!", + "text": "Be careful!!!\ntidymodels has always contained some “hidden guardrails” that should prevent practitioners from making subtle (but consequential) methodological mistakes.\n\nFeature selection is a good example. Based on the literature, it is easily done wrong.\n\nThe selection process should take place inside a resampling loop so that the workflow does not overfit the predictors.\n\nA&M (2002), AML4TD, FES" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", - "href": "slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "slides/advanced-07-feature-selection.html#important", + "href": "slides/advanced-07-feature-selection.html#important", + "title": "7 - Feature selection", + "section": "IMPORTANT ", + "text": "IMPORTANT \nWe released two packages this year that enable supervised feature selection:\n\nfiltro: low-level scoring methods for predictors (e.g., importance).\nimportant: tools for permutation importance and recipes steps for supervised feature selection.\n\n\nLet’s look at the help page for important::step_predictor_best()." }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "href": "slides/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "title": "3 - Tuning Hyperparameters", - "section": "Tagging parameters for tuning ", - "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" + "objectID": "slides/advanced-07-feature-selection.html#k-nearest-neighbors", + "href": "slides/advanced-07-feature-selection.html#k-nearest-neighbors", + "title": "7 - Feature selection", + "section": "K-nearest neighbors ", + "text": "K-nearest neighbors \n\nrec <-\n recipe(class ~ ., data = sim_train) |>\n step_predictor_best(\n all_predictors(),\n score = \"imp_rf\",\n prop_terms = tune(),\n id = \"filter\"\n ) |>\n step_normalize(all_numeric_predictors())\n \nknn_spec <- \n nearest_neighbor(neighbors = tune(), weight_func = tune()) |> \n set_mode(\"classification\")\n \nthrsh_tlr <-\n tailor() |>\n adjust_probability_threshold(threshold = tune())" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", - "href": "slides/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", - "title": "3 - Tuning Hyperparameters", - "section": "Optimizing the hash features ", - "text": "Optimizing the hash features \nOur new recipe is:\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) |>\n step_dummy_hash(company, num_terms = tune(\"company hash\")) |>\n step_zv(all_predictors())\n\n\nWe will be using a tree-based model in a minute.\n\nThe other categorical predictors are left as-is.\nThat’s why there is no step_dummy()." + "objectID": "slides/advanced-07-feature-selection.html#setup-the-workflow", + "href": "slides/advanced-07-feature-selection.html#setup-the-workflow", + "title": "7 - Feature selection", + "section": "Setup the workflow ", + "text": "Setup the workflow \n\nknn_wflow <- workflow(rec, knn_spec, thrsh_tlr)\n\nknn_param <-\n knn_wflow |>\n extract_parameter_set_dials() |>\n update(\n threshold = threshold(c(0.001, 0.1)),\n neighbors = neighbors(c(1, 50))\n )" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#boosted-trees", - "href": "slides/advanced-03-tuning-hyperparameters.html#boosted-trees", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Trees", - "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved and new samples are predicted using a weighted average of the votes of each tree in the ensemble.\n\nWe’ll focus on the popular lightgbm implementation." + "objectID": "slides/advanced-07-feature-selection.html#tuning-results", + "href": "slides/advanced-07-feature-selection.html#tuning-results", + "title": "7 - Feature selection", + "section": "Tuning results ", + "text": "Tuning results \n\ncls_mtr <- metric_set(brier_class, roc_auc, sensitivity, specificity)\nctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nset.seed(12)\nknn_res <-\n knn_wflow |>\n tune_grid(\n resamples = sim_rs,\n grid = 50,\n control = ctrl,\n metrics = cls_mtr,\n param_info = knn_param\n )" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "href": "slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands)\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))" + "objectID": "slides/advanced-07-feature-selection.html#grid-results", + "href": "slides/advanced-07-feature-selection.html#grid-results", + "title": "7 - Feature selection", + "section": "Grid results ", + "text": "Grid results \n\nautoplot(knn_res)" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "href": "slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nTBH it is usually not difficult to optimize these models.\n\nOften, there are multiple candidate tuning parameter combinations that have very good results.\n\nTo demonstrate simple concepts, we’ll look at optimizing the number of trees in the ensemble (between 1 and 100) and the learning rate (\\(10^{-5}\\) to \\(10^{-1}\\))." + "objectID": "slides/advanced-07-feature-selection.html#brier-results", + "href": "slides/advanced-07-feature-selection.html#brier-results", + "title": "7 - Feature selection", + "section": "Brier results ", + "text": "Brier results \n\nautoplot(knn_res, metric = \"brier_class\") + \n facet_grid(. ~ name, scale = \"free_x\")" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "href": "slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters ", - "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)" + "objectID": "slides/advanced-07-feature-selection.html#roc-curve-results", + "href": "slides/advanced-07-feature-selection.html#roc-curve-results", + "title": "7 - Feature selection", + "section": "ROC curve results ", + "text": "ROC curve results \n\nautoplot(knn_res, metric = \"roc_auc\") + \n facet_grid(. ~ name, scale = \"free_x\")" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "href": "slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "title": "3 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "slides/advanced-07-feature-selection.html#sensitivityspecificity-results", + "href": "slides/advanced-07-feature-selection.html#sensitivityspecificity-results", + "title": "7 - Feature selection", + "section": "Sensitivity/Specificity results ", + "text": "Sensitivity/Specificity results \n\nautoplot(knn_res, metric = c(\"sensitivity\", \"specificity\"))" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#grid-search", - "href": "slides/advanced-03-tuning-hyperparameters.html#grid-search", - "title": "3 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" + "objectID": "slides/advanced-07-feature-selection.html#fit-the-model-and-get-filter-information", + "href": "slides/advanced-07-feature-selection.html#fit-the-model-and-get-filter-information", + "title": "7 - Feature selection", + "section": "Fit the model and get filter information ", + "text": "Fit the model and get filter information \n\nknn_fit <- fit_best(knn_res, metric = \"brier_class\")\nfilter_info <-\n knn_fit |>\n extract_recipe() |>\n tidy(id = \"filter\")\n\nfilter_info\n#> # A tibble: 30 × 4\n#> terms removed score id \n#> <chr> <lgl> <dbl> <chr> \n#> 1 predictor_01 TRUE 0.00152 filter\n#> 2 predictor_02 TRUE 0.00170 filter\n#> 3 predictor_03 TRUE -0.0000460 filter\n#> 4 predictor_04 TRUE 0.000968 filter\n#> 5 predictor_05 TRUE -0.0000626 filter\n#> 6 predictor_06 TRUE 0.000126 filter\n#> 7 predictor_07 TRUE 0.000779 filter\n#> 8 predictor_08 TRUE 0.00160 filter\n#> 9 predictor_09 TRUE 0.00218 filter\n#> 10 predictor_10 TRUE -0.000302 filter\n#> # ℹ 20 more rows" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#grid-search-1", - "href": "slides/advanced-03-tuning-hyperparameters.html#grid-search-1", - "title": "3 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nIn reality we would probably sample the space more densely:" + "objectID": "slides/advanced-07-feature-selection.html#the-truth-about-our-data", + "href": "slides/advanced-07-feature-selection.html#the-truth-about-our-data", + "title": "7 - Feature selection", + "section": "The truth about our data", + "text": "The truth about our data\nThe data were simulated and 15 out of 30 predictors were uninformative (and highly correlated). How did we do?\n\n\n\n\n\n\n\n\n\n\nnoise\nreal\n\n\n\n\nkept\n0\n2\n\n\nremoved\n15\n13\n\n\n\n\n\n\n\nselection sensitivity: 13.3%\nselection specificity: 100%\n\nIt was good at removing noise but not keeping the real predictors." }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#iterative-search", - "href": "slides/advanced-03-tuning-hyperparameters.html#iterative-search", - "title": "3 - Tuning Hyperparameters", - "section": "Iterative Search", - "text": "Iterative Search\nWe could start with a few points and search the space:" + "objectID": "slides/advanced-07-feature-selection.html#random-forest-importance-scores", + "href": "slides/advanced-07-feature-selection.html#random-forest-importance-scores", + "title": "7 - Feature selection", + "section": "Random forest importance scores", + "text": "Random forest importance scores\n\n\n# A \"truth\" column was added\nfilter_info |>\n mutate(\n terms = factor(terms),\n terms = reorder(terms, score)\n ) |>\n ggplot(\n aes(x = score, \n y = terms, \n fill = truth)\n ) +\n geom_bar(stat = \"identity\") + \n labs(x = \"RF Importance\", y = NULL) + \n scale_fill_brewer(palette = \"Set2\")" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#parameters", - "href": "slides/advanced-03-tuning-hyperparameters.html#parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Parameters", - "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" + "objectID": "slides/advanced-07-feature-selection.html#the-simulation", + "href": "slides/advanced-07-feature-selection.html#the-simulation", + "title": "7 - Feature selection", + "section": "The simulation", + "text": "The simulation\nThe simulation system is documented here with method = \"caret\". The two most important predictors being retained correspond to:\n\n# In logit units: \n- 4 * two_factor_1 + 4 * two_factor_2 + 2 * two_factor_1 * two_factor_2 \n\n\nMost of the others are small linear effects and tree-based models are not great at modeling those.\n\nAlso, the noise predictors were simulated to have fairly high correlations with one another. That can often compromise random forest importance scores." }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#different-types-of-grids", - "href": "slides/advanced-03-tuning-hyperparameters.html#different-types-of-grids", - "title": "3 - Tuning Hyperparameters", - "section": "Different types of grids ", - "text": "Different types of grids \n\n\n\n\n\n\n\n\n\nSpace-filling designs (SFD) attempt to cover the parameter space without redundant candidates. We recommend these the most." + "objectID": "slides/advanced-07-feature-selection.html#other-steps", + "href": "slides/advanced-07-feature-selection.html#other-steps", + "title": "7 - Feature selection", + "section": "Other steps", + "text": "Other steps\nThe important package has two other feature selection steps that can be used with multiple scores:\n\n\nstep_predictor_retain(): choose predictors based on a logical statement. Example:\n\nimp_rf > 2 & cor_pearson >= 0.75\n\n\n\nstep_predictor_desirability(): choose multiple scores to compute then use desirability functions to rank them:\n\ndesirability(\n maximize(correlation),\n maximize(imp_rf)\n)" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#create-a-grid", - "href": "slides/advanced-03-tuning-hyperparameters.html#create-a-grid", - "title": "3 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\nlgbm_wflow |> \n extract_parameter_set_dials()\n\n# Individual functions: \ntrees()\nlearn_rate()\n\n\nA parameter set can be updated (e.g. to change the ranges)." + "objectID": "slides/advanced-07-feature-selection.html#manual-approach", + "href": "slides/advanced-07-feature-selection.html#manual-approach", + "title": "7 - Feature selection", + "section": "Manual approach ", + "text": "Manual approach \nWe already have our fitted model and, if we are happy with it:\n\ntest_pred <- augment(knn_fit, sim_test)\ntest_pred |> cls_mtr(class, estimate = .pred_class, .pred_event)\n#> # A tibble: 4 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.982 \n#> 2 specificity binary 0.874 \n#> 3 brier_class binary 0.0246\n#> 4 roc_auc binary 0.981\n\n\nResampling estimates:\n\n#> # A tibble: 4 × 4\n#> .metric mean n std_err\n#> <chr> <dbl> <int> <dbl>\n#> 1 sensitivity 0.958 10 0.0154 \n#> 2 specificity 0.867 10 0.00948\n#> 3 brier_class 0.0349 10 0.00201\n#> 4 roc_auc 0.968 10 0.00714" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#create-a-grid-1", - "href": "slides/advanced-03-tuning-hyperparameters.html#create-a-grid-1", - "title": "3 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\ngrid <- \n lgbm_wflow |> \n extract_parameter_set_dials() |> \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 7.50e- 6 574 574\n#> 2 84 1.78e- 5 2048 2298\n#> 3 167 5.62e-10 1824 912\n#> 4 250 4.22e- 5 3250 512\n#> 5 334 1.78e- 8 512 2896\n#> 6 417 1.33e- 3 322 1625\n#> 7 500 1 e- 1 1448 1149\n#> 8 584 1 e- 7 1290 256\n#> 9 667 2.37e-10 456 724\n#> 10 750 1.78e- 2 645 322\n#> # ℹ 15 more rows" + "objectID": "slides/advanced-07-feature-selection.html#checking-approximate-calibration", + "href": "slides/advanced-07-feature-selection.html#checking-approximate-calibration", + "title": "7 - Feature selection", + "section": "Checking (Approximate) Calibration ", + "text": "Checking (Approximate) Calibration \n\n\n\ntest_pred|>\n cal_plot_windowed(\n truth = class,\n estimate = .pred_event,\n window_size = 0.2,\n step_size = 0.025,\n )\n\n\nLooks alright. The small effective sample size (57 events) makes it pretty noisy." }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#your-turn", - "href": "slides/advanced-03-tuning-hyperparameters.html#your-turn", - "title": "3 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n−+\n03:00" + "objectID": "slides/advanced-07-feature-selection.html#automated-approach", + "href": "slides/advanced-07-feature-selection.html#automated-approach", + "title": "7 - Feature selection", + "section": "Automated approach ", + "text": "Automated approach \nSimilar to fit_best(), there is a convenience function that can be used to get the final model and the test set results.\n\nWe have to start with a finalized workflow (i.e., no tune() values):\n\nknn_best <- select_best(knn_res, metric = \"brier_class\")\nknn_last_wflow <- finalize_workflow(knn_wflow, knn_best)" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", - "href": "slides/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", - "title": "3 - Tuning Hyperparameters", - "section": "Create a regular grid ", - "text": "Create a regular grid \n\ngrid <- \n lgbm_wflow |> \n extract_parameter_set_dials() |> \n grid_regular(levels = 4)\n\ngrid\n#> # A tibble: 256 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.0000000001 256 256\n#> 2 667 0.0000000001 256 256\n#> 3 1333 0.0000000001 256 256\n#> 4 2000 0.0000000001 256 256\n#> 5 1 0.0000001 256 256\n#> 6 667 0.0000001 256 256\n#> 7 1333 0.0000001 256 256\n#> 8 2000 0.0000001 256 256\n#> 9 1 0.0001 256 256\n#> 10 667 0.0001 256 256\n#> # ℹ 246 more rows" + "objectID": "slides/advanced-07-feature-selection.html#automated-approach-1", + "href": "slides/advanced-07-feature-selection.html#automated-approach-1", + "title": "7 - Feature selection", + "section": "Automated approach", + "text": "Automated approach\nlast_fit() uses the original split object to fit, predict, and measure the model using the test set:\n\nknn_test_res <- \n knn_last_wflow |> \n last_fit(sim_split, metrics = cls_mtr)\n \nknn_test_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [1499/501]> train/test split <tibble> <tibble> <tibble> <workflow>" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#your-turn-1", - "href": "slides/advanced-03-tuning-hyperparameters.html#your-turn-1", - "title": "3 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\n\nWhat advantage would a regular grid have?" + "objectID": "slides/advanced-07-feature-selection.html#automated-approach-2", + "href": "slides/advanced-07-feature-selection.html#automated-approach-2", + "title": "7 - Feature selection", + "section": "Automated approach", + "text": "Automated approach\nWe can pick out the parts that we want:\n\nknn_final_fit <- knn_test_res |> extract_workflow()\nknn_test_pred <- knn_test_res |> collect_predictions()\nknn_test_mtr <- knn_test_res |> collect_metrics()\n\nknn_test_mtr\n#> # A tibble: 4 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 sensitivity binary 0.982 pre0_mod0_post0\n#> 2 specificity binary 0.874 pre0_mod0_post0\n#> 3 brier_class binary 0.0246 pre0_mod0_post0\n#> 4 roc_auc binary 0.981 pre0_mod0_post0\n\nEasy peasy!" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", - "href": "slides/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", - "title": "3 - Tuning Hyperparameters", - "section": "Update parameter ranges ", - "text": "Update parameter ranges \n\nlgbm_param <- \n lgbm_wflow |> \n extract_parameter_set_dials() |> \n update(trees = trees(c(1L, 100L)),\n learn_rate = learn_rate(c(-5, -1)))\n\ngrid <- \n lgbm_param |> \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.00147 574 574\n#> 2 5 0.00215 2048 2298\n#> 3 9 0.0000215 1824 912\n#> 4 13 0.00316 3250 512\n#> 5 17 0.0001 512 2896\n#> 6 21 0.0147 322 1625\n#> 7 25 0.1 1448 1149\n#> 8 29 0.000215 1290 256\n#> 9 34 0.0000147 456 724\n#> 10 38 0.0464 645 322\n#> # ℹ 15 more rows" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#getting-set-up", + "href": "slides/advanced-05-feature-engineering-part-two.html#getting-set-up", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Getting set up ", + "text": "Getting set up \n\nlibrary(tidymodels)\nlibrary(embed)\nlibrary(extrasteps)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#the-results", - "href": "slides/advanced-03-tuning-hyperparameters.html#the-results", - "title": "3 - Tuning Hyperparameters", - "section": "The results ", - "text": "The results \n\n\ngrid |> \n ggplot(aes(trees, learn_rate)) +\n geom_point(size = 4) +\n scale_y_log10()\n\n\n\n\n\n\n\n\n\nNote that the learning rates are uniform on the log-10 scale and this shows 2 of 4 dimensions." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#hotel-rates-data-set", + "href": "slides/advanced-05-feature-engineering-part-two.html#hotel-rates-data-set", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Hotel rates data set", + "text": "Hotel rates data set\nRegression data set for predicting the average daily rate for a room, for “Resort Hotel”. The agent and company use random names.\n\nglimpse(hotel_rates)\n#> Rows: 15,402\n#> Columns: 28\n#> $ avg_price_per_room <dbl> 110.00, 74.00, 81.90, 81.00, 112.20, 90…\n#> $ lead_time <dbl> 241, 273, 248, 236, 243, 267, 94, 10, 1…\n#> $ stays_in_weekend_nights <dbl> 0, 2, 2, 2, 4, 2, 4, 0, 0, 0, 0, 0, 0, …\n#> $ stays_in_week_nights <dbl> 1, 5, 5, 5, 10, 5, 7, 1, 1, 1, 1, 1, 1,…\n#> $ adults <dbl> 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, …\n#> $ children <dbl> 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, …\n#> $ babies <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ meal <fct> bed_and_breakfast, bed_and_breakfast, b…\n#> $ country <fct> prt, aus, gbr, prt, gbr, null, prt, esp…\n#> $ market_segment <fct> online_travel_agent, offline_travel_age…\n#> $ distribution_channel <fct> ta_to, ta_to, ta_to, ta_to, ta_to, ta_t…\n#> $ is_repeated_guest <dbl> 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, …\n#> $ previous_cancellations <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ previous_bookings_not_canceled <dbl> 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, …\n#> $ reserved_room_type <fct> a, a, a, a, a, a, f, e, h, a, a, g, a, …\n#> $ assigned_room_type <fct> c, a, c, a, a, a, f, f, h, e, e, g, e, …\n#> $ booking_changes <dbl> 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, …\n#> $ agent <fct> devin_rivera_borrego, lia_nauth, jawhar…\n#> $ company <fct> not_applicable, not_applicable, not_app…\n#> $ days_in_waiting_list <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ customer_type <fct> transient, transient_party, transient, …\n#> $ required_car_parking_spaces <dbl> 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, …\n#> $ total_of_special_requests <dbl> 1, 0, 0, 2, 0, 0, 1, 1, 0, 2, 2, 0, 2, …\n#> $ arrival_date <date> 2016-07-02, 2016-07-02, 2016-07-02, 20…\n#> $ arrival_date_num <dbl> 2016.5, 2016.5, 2016.5, 2016.5, 2016.5,…\n#> $ near_christmas <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ near_new_years <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ historical_adr <dbl> 104.9811, 104.9811, 104.9811, 104.9811,…\n\n\nAntonio, N., de Almeida, A., and Nunes, L. (2019). Hotel booking demand datasets. Data in Brief, 22, 41-49." }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", - "href": "slides/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Choosing tuning parameters ", - "text": "Choosing tuning parameters \nLet’s take our previous model and tune more parameters:\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)\n\n# Update the feature hash ranges (log-2 units)\nlgbm_param <-\n lgbm_wflow |>\n extract_parameter_set_dials() |>\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#hotel-data-splitting", + "href": "slides/advanced-05-feature-engineering-part-two.html#hotel-data-splitting", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Hotel data splitting ", + "text": "Hotel data splitting \nGenerally, you should always do data splitting. We are doing it here explicitly because some artifacts of splitting data become useful later on.\n\nset.seed(1234)\nhotel_split <- initial_split(hotel_rates)\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#grid-search-3", - "href": "slides/advanced-03-tuning-hyperparameters.html#grid-search-3", - "title": "3 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE)\n\nlgbm_res <-\n lgbm_wflow |>\n tune_grid(\n resamples = hotel_rs,\n grid = 25,\n # The options below are not required by default\n param_info = lgbm_param, \n control = ctrl,\n metrics = reg_metrics\n )\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn", + "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Your turn", + "text": "Your turn\n\nLoad and explore the hotel_train data\nComes loaded with the modeldata package\n\n\n\n−+\n05:00" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#grid-search-4", - "href": "slides/advanced-03-tuning-hyperparameters.html#grid-search-4", - "title": "3 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nlgbm_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [3372/377]> Fold01 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,425 × 9]>\n#> 2 <split [3373/376]> Fold02 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 3 <split [3373/376]> Fold03 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 4 <split [3373/376]> Fold04 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 5 <split [3373/376]> Fold05 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 6 <split [3374/375]> Fold06 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,375 × 9]>\n#> 7 <split [3375/374]> Fold07 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,350 × 9]>\n#> 8 <split [3376/373]> Fold08 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 9 <split [3376/373]> Fold09 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 10 <split [3376/373]> Fold10 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#nonlinear-predictors", + "href": "slides/advanced-05-feature-engineering-part-two.html#nonlinear-predictors", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Nonlinear predictors", + "text": "Nonlinear predictors" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#grid-results", - "href": "slides/advanced-03-tuning-hyperparameters.html#grid-results", - "title": "3 - Tuning Hyperparameters", - "section": "Grid results ", - "text": "Grid results \n\nautoplot(lgbm_res)" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#splines-1", + "href": "slides/advanced-05-feature-engineering-part-two.html#splines-1", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Splines", + "text": "Splines\nIt is a way to transform a single numeric predictor into multiple numeric predictors, with the hope that the new numeric predictors are more linearly related to the outcome.\nMostly needed with linear models, but it should rarely hurt to use it.\nIf you’ve ever used geom_smooth() you have seen splines in action.\n\nFEAZ FES" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#tuning-results", - "href": "slides/advanced-03-tuning-hyperparameters.html#tuning-results", - "title": "3 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res)\n#> # A tibble: 50 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1500 21 0.00000000316 8 60 mae standard 53.2 10 0.427 pre01_mod19_post0\n#> 2 1500 21 0.00000000316 8 60 rsq standard 0.809 10 0.00661 pre01_mod19_post0\n#> 3 917 16 0.0422 9 39 mae standard 9.85 10 0.150 pre02_mod12_post0\n#> 4 917 16 0.0422 9 39 rsq standard 0.946 10 0.00362 pre02_mod12_post0\n#> 5 584 13 0.0000001 10 10 mae standard 53.2 10 0.427 pre03_mod08_post0\n#> 6 584 13 0.0000001 10 10 rsq standard 0.810 10 0.00745 pre03_mod08_post0\n#> 7 167 19 0.00000133 12 143 mae standard 53.2 10 0.426 pre04_mod03_post0\n#> 8 167 19 0.00000133 12 143 rsq standard 0.811 10 0.00698 pre04_mod03_post0\n#> 9 1583 33 0.0000422 14 12 mae standard 50.3 10 0.405 pre05_mod20_post0\n#> 10 1583 33 0.0000422 14 12 rsq standard 0.817 10 0.00768 pre05_mod20_post0\n#> # ℹ 40 more rows" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#splines-explained", + "href": "slides/advanced-05-feature-engineering-part-two.html#splines-explained", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Splines explained", + "text": "Splines explained\nA spline is a piecewise polynomial function.\nWe have 2 main parameters to worry about. Number of knots and the polynomial degree.\nThe domain of the predictor is split into k regions, with a knot between each, and a polynomial function is fit within each region, under the constraint that they touch each other at the knot." }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#tuning-results-1", - "href": "slides/advanced-03-tuning-hyperparameters.html#tuning-results-1", - "title": "3 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res, summarize = FALSE)\n#> # A tibble: 500 × 10\n#> id trees min_n learn_rate `agent hash` `company hash` .metric .estimator .estimate .config \n#> <chr> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 Fold01 1500 21 0.00000000316 8 60 mae standard 51.8 pre01_mod19_post0\n#> 2 Fold01 1500 21 0.00000000316 8 60 rsq standard 0.805 pre01_mod19_post0\n#> 3 Fold02 1500 21 0.00000000316 8 60 mae standard 52.1 pre01_mod19_post0\n#> 4 Fold02 1500 21 0.00000000316 8 60 rsq standard 0.800 pre01_mod19_post0\n#> 5 Fold03 1500 21 0.00000000316 8 60 mae standard 52.2 pre01_mod19_post0\n#> 6 Fold03 1500 21 0.00000000316 8 60 rsq standard 0.783 pre01_mod19_post0\n#> 7 Fold04 1500 21 0.00000000316 8 60 mae standard 51.7 pre01_mod19_post0\n#> 8 Fold04 1500 21 0.00000000316 8 60 rsq standard 0.818 pre01_mod19_post0\n#> 9 Fold05 1500 21 0.00000000316 8 60 mae standard 55.2 pre01_mod19_post0\n#> 10 Fold05 1500 21 0.00000000316 8 60 rsq standard 0.845 pre01_mod19_post0\n#> # ℹ 490 more rows" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-1-degree-1", + "href": "slides/advanced-05-feature-engineering-part-two.html#knots-1-degree-1", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "knots: 1, degree: 1", + "text": "knots: 1, degree: 1" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", - "href": "slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", - "title": "3 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \n\nshow_best(lgbm_res, metric = \"rsq\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1250 9 0.1 107 80 rsq standard 0.947 10 0.00395 pre19_mod16_post0\n#> 2 917 16 0.0422 9 39 rsq standard 0.946 10 0.00362 pre02_mod12_post0\n#> 3 1666 30 0.00750 124 19 rsq standard 0.943 10 0.00309 pre20_mod21_post0\n#> 4 2000 24 0.00133 25 124 rsq standard 0.919 10 0.00383 pre09_mod25_post0\n#> 5 500 25 0.00316 52 8 rsq standard 0.899 10 0.00435 pre14_mod07_post0" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-2-degree-1", + "href": "slides/advanced-05-feature-engineering-part-two.html#knots-2-degree-1", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "knots: 2, degree: 1", + "text": "knots: 2, degree: 1" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "href": "slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "title": "3 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nlgbm_best <- select_best(lgbm_res, metric = \"mae\")\nlgbm_best\n#> # A tibble: 1 × 6\n#> trees min_n learn_rate `agent hash` `company hash` .config \n#> <int> <int> <dbl> <int> <int> <chr> \n#> 1 1250 9 0.1 107 80 pre19_mod16_post0" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-1", + "href": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-1", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "knots: 5, degree: 1", + "text": "knots: 5, degree: 1" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#checking-calibration", - "href": "slides/advanced-03-tuning-hyperparameters.html#checking-calibration", - "title": "3 - Tuning Hyperparameters", - "section": "Checking Calibration ", - "text": "Checking Calibration \n\n\nlibrary(probably)\nlgbm_res |>\n collect_predictions(\n parameters = lgbm_best\n ) |>\n cal_plot_regression(\n truth = avg_price_per_room,\n estimate = .pred\n )" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-2", + "href": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-2", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "knots: 5, degree: 2", + "text": "knots: 5, degree: 2" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#running-in-parallel", - "href": "slides/advanced-03-tuning-hyperparameters.html#running-in-parallel", - "title": "3 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\n\nlibrary(future)\nplan(multisession, workers = cores)\n\n# Now call `tune_grid()`!" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-3", + "href": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-3", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "knots: 5, degree: 3", + "text": "knots: 5, degree: 3" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", - "href": "slides/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", - "title": "3 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-9-degree-3", + "href": "slides/advanced-05-feature-engineering-part-two.html#knots-9-degree-3", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "knots: 9, degree: 3", + "text": "knots: 9, degree: 3" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", - "href": "slides/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", - "title": "3 - Tuning Hyperparameters", - "section": "The ‘future’ of parallel processing", - "text": "The ‘future’ of parallel processing\nWe have relied on the foreach package for parallel processing.\nWe will start the transition to using the future package in the upcoming version of the tune package (version 1.3.0).\nThere will be a period of backward compatibility where you can still use foreach with future via the doFuture package. After that, the transition to future will occur.\nOverall, there will be minimal changes to your code." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#b-spline-features-visualized---degree-3", + "href": "slides/advanced-05-feature-engineering-part-two.html#b-spline-features-visualized---degree-3", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "B-Spline features visualized - degree: 3", + "text": "B-Spline features visualized - degree: 3" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "href": "slides/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "title": "3 - Tuning Hyperparameters", - "section": "Early stopping for boosted trees", - "text": "Early stopping for boosted trees\nWe have directly optimized the number of trees as a tuning parameter.\nInstead we could\n\nSet the number of trees to a single large number.\nStop adding trees when performance gets worse.\n\nThis is known as “early stopping” and there is a parameter for that: stop_iter.\nEarly stopping has a potential to decrease the tuning time." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#splines-as-numbers", + "href": "slides/advanced-05-feature-engineering-part-two.html#splines-as-numbers", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Splines as numbers", + "text": "Splines as numbers\n\n\n\n\n\n\n\n\n\n\n\n\n\n\narrival_date_num\nSpline Feature 1\nSpline Feature 2\nSpline Feature 3\nSpline Feature 4\nSpline Feature 5\nSpline Feature 6\n\n\n\n\n2017.619\n0.00\n0.00\n0.00\n0.03\n0.35\n0.62\n\n\n2016.844\n0.15\n0.59\n0.26\n0.00\n0.00\n0.00\n\n\n2016.702\n0.51\n0.40\n0.05\n0.00\n0.00\n0.00\n\n\n2017.077\n0.00\n0.19\n0.67\n0.14\n0.00\n0.00\n\n\n2016.861\n0.13\n0.58\n0.29\n0.00\n0.00\n0.00\n\n\n2017.019\n0.00\n0.30\n0.62\n0.07\n0.00\n0.00\n\n\n2017.123\n0.00\n0.11\n0.66\n0.23\n0.00\n0.00" }, { - "objectID": "slides/advanced-03-tuning-hyperparameters.html#your-turn-2", - "href": "slides/advanced-03-tuning-hyperparameters.html#your-turn-2", - "title": "3 - Tuning Hyperparameters", + "objectID": "slides/advanced-05-feature-engineering-part-two.html#splines-pros-and-cons", + "href": "slides/advanced-05-feature-engineering-part-two.html#splines-pros-and-cons", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Splines pros and cons", + "text": "Splines pros and cons\n\n\nPros\n\nfast\neasy to use\nsemi-interpretable\n\n\nCons\n\nadds more columns\nwill need to select the number of columns\ncan be messy outside the range" + }, + { + "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn-1", + "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn-1", + "title": "5 - Feature engineering: splines, target encoding and dates", "section": "Your turn", - "text": "Your turn\n\n\nSet trees = 2000 and tune the stop_iter parameter.\nNote that you will need to regenerate lgbm_param with your new workflow!\n\n\n\n−+\n10:00" + "text": "Your turn\n\nApply B-splines to some variables using step_spline_b()\n\n\n\n−+\n03:00" }, { - "objectID": "slides/advanced-01-introduction.html#who-are-you", - "href": "slides/advanced-01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML\nYou have used some tidymodels packages\nYou have some experience with evaluating statistical models using resampling techniques" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#factors-with-many-categories", + "href": "slides/advanced-05-feature-engineering-part-two.html#factors-with-many-categories", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Factors with many categories", + "text": "Factors with many categories\n\n\n\nhotel_train |>\n count(country)\n#> # A tibble: 88 × 2\n#> country n\n#> <fct> <int>\n#> 1 ago 7\n#> 2 and 1\n#> 3 are 3\n#> 4 arg 14\n#> 5 aus 31\n#> 6 aut 76\n#> 7 aze 2\n#> 8 bel 184\n#> 9 bgr 2\n#> 10 bhs 1\n#> # ℹ 78 more rows\n\n\n\nhotel_train |>\n count(company)\n#> # A tibble: 157 × 2\n#> company n\n#> <fct> <int>\n#> 1 abdou_llc 2\n#> 2 afework_llc 5\n#> 3 alston_pbc 3\n#> 4 battle_llc 23\n#> 5 bennett_and_company 3\n#> 6 berhanu_pbc 53\n#> 7 biggers_llc 4\n#> 8 blasingime_llc 1\n#> 9 boddy_llc 16\n#> 10 boles_pbc 3\n#> # ℹ 147 more rows\n\n\n\nhotel_train |>\n count(agent)\n#> # A tibble: 119 × 2\n#> agent n\n#> <fct> <int>\n#> 1 aaron_marquez 2\n#> 2 alexander_drake 1117\n#> 3 allen_her 1\n#> 4 anas_el_bashir 1\n#> 5 araseli_billy 1\n#> 6 arhab_al_islam 7\n#> 7 audray_tucker 38\n#> 8 bernice_baltierra 35\n#> 9 betzy_rodriguez 66\n#> 10 brayan_guerrero 2\n#> # ℹ 109 more rows" }, { - "objectID": "slides/advanced-01-introduction.html#who-are-tidymodels", - "href": "slides/advanced-01-introduction.html#who-are-tidymodels", - "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#how-do-we-handle-them", + "href": "slides/advanced-05-feature-engineering-part-two.html#how-do-we-handle-them", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "How do we handle them?", + "text": "How do we handle them?\n\nWe could:\n\nMake the full set of indicator variables 😳\nLump agents and companies that rarely occur into an “other” group\nUse feature hashing to create a smaller set of indicator variables\nUse target encoding to replace the county, agent, and company columns with the estimated effect of that predictor" }, { - "objectID": "slides/advanced-01-introduction.html#section-2", - "href": "slides/advanced-01-introduction.html#section-2", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-1", + "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-1", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Target encoding", + "text": "Target encoding\nTarget encoding (also called mean encoding, likelihood encoding, impact encoding, or effect encoding) is a supervised trained method that turns a single categorical predictor into a single numeric predictor.\nIt is often used to deal with categorical predictors with many levels, although it works regardless.\nSince it uses the outcome to train it, you need to make sure to use cross-validation to avoid overfitting.\n\nFEAZ FES" }, { - "objectID": "slides/advanced-01-introduction.html#section-3", - "href": "slides/advanced-01-introduction.html#section-3", - "title": "1 - Introduction", - "section": "👀", - "text": "👀\n\n\nhttps://workshops.tidymodels.org" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-motivation", + "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-motivation", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Target encoding motivation", + "text": "Target encoding motivation\nYou have a numeric outcome and a categorical predictor. And you want to transform each value of the categorical predictor into a value that best represents the outcome?\n\n\n\nWe calculate the mean of the outcome within each level of the predictor, and use that as the new value.\n\n\n\n\n\n\nCaution\n\n\nDon’t do just this! We are building up the method one thing at a time. Unregularized target encoding is really prone to overfitting.\n\n\n\n\n\nhotel_train |>\n summarise(\n mean = mean(avg_price_per_room),\n .by = agent\n )\n#> # A tibble: 119 × 2\n#> agent mean\n#> <fct> <dbl>\n#> 1 alexander_drake 144. \n#> 2 kaylae_maxedon 62.5\n#> 3 michael_mcdole 60.9\n#> 4 devin_rivera_borrego 126. \n#> 5 james_richards 78.6\n#> 6 estela_bonilla 41.9\n#> 7 charles_najera 109. \n#> 8 reema_el_tamer 118. \n#> 9 jawhara_al_azad 90.1\n#> 10 not_applicable 84.1\n#> # ℹ 109 more rows" }, { - "objectID": "slides/advanced-01-introduction.html#tentative-plan-for-this-workshop", - "href": "slides/advanced-01-introduction.html#tentative-plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Tentative plan for this workshop", - "text": "Tentative plan for this workshop\n\nFeature engineering with recipes\nModel optimization by tuning\n\nGrid search\nRacing\nIterative methods\n\nExtras (time permitting)\n\nEffect encodings\nA case study" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-handling-unseen-levels", + "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-handling-unseen-levels", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Target encoding handling unseen levels", + "text": "Target encoding handling unseen levels\n\n\n\n\n\n\nCaution\n\n\nDon’t look at the testing data set. This is done for educational purposes.\n\n\n\n\n\n\nhotel_train |>\n count(agent, .drop = FALSE)\n#> # A tibble: 174 × 2\n#> agent n\n#> <fct> <int>\n#> 1 aaron_marquez 2\n#> 2 aayaat_al_farran 0\n#> 3 alanah_cook 0\n#> 4 alexander_drake 1117\n#> 5 allen_her 1\n#> 6 amirah_christian 0\n#> 7 anas_el_bashir 1\n#> 8 anna_beltran_moreno 0\n#> 9 anna_choi 0\n#> 10 araseli_billy 1\n#> # ℹ 164 more rows\n\n\n\nhotel_test |>\n count(agent, .drop = FALSE)\n#> # A tibble: 174 × 2\n#> agent n\n#> <fct> <int>\n#> 1 aaron_marquez 1\n#> 2 aayaat_al_farran 0\n#> 3 alanah_cook 0\n#> 4 alexander_drake 367\n#> 5 allen_her 1\n#> 6 amirah_christian 0\n#> 7 anas_el_bashir 0\n#> 8 anna_beltran_moreno 0\n#> 9 anna_choi 0\n#> 10 araseli_billy 1\n#> # ℹ 164 more rows" }, { - "objectID": "slides/advanced-01-introduction.html#section-4", - "href": "slides/advanced-01-introduction.html#section-4", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free): TODO-ADD-LATER" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-handling-unseen-levels-1", + "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-handling-unseen-levels-1", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Target encoding handling unseen levels", + "text": "Target encoding handling unseen levels\n\n\nCalculate the global mean of the outcome and use it for cases that aren’t seen in the training data set.\n\n\nmean(hotel_train$avg_price_per_room)\n#> [1] 104.6039\n\n\n\nhotel_train |>\n summarise(\n mean = mean(avg_price_per_room),\n .by = agent\n )\n#> # A tibble: 119 × 2\n#> agent mean\n#> <fct> <dbl>\n#> 1 alexander_drake 144. \n#> 2 kaylae_maxedon 62.5\n#> 3 michael_mcdole 60.9\n#> 4 devin_rivera_borrego 126. \n#> 5 james_richards 78.6\n#> 6 estela_bonilla 41.9\n#> 7 charles_najera 109. \n#> 8 reema_el_tamer 118. \n#> 9 jawhara_al_azad 90.1\n#> 10 not_applicable 84.1\n#> # ℹ 109 more rows" }, { - "objectID": "slides/advanced-01-introduction.html#lets-install-some-packages", - "href": "slides/advanced-01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"lightgbm\", \"lme4\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nAlso, you should install the newest version of the dials package (version 1.3.0). To check this, you can run:\n\nrlang::check_installed(\"dials\", version = \"1.3.0\")" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#how-do-we-handle-low-counts", + "href": "slides/advanced-05-feature-engineering-part-two.html#how-do-we-handle-low-counts", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "How do we handle low counts?", + "text": "How do we handle low counts?\n\n\nSome of the levels have very low counts. We can’t have the same confidence in those means as the means calculated on high counts.\nWe use the global mean to account for 0 occurrences. Let us adjust the calculated mean by the global mean depending on the counts.\n\n\nhotel_train |>\n summarise(\n mean = mean(avg_price_per_room),\n n = n(),\n .by = agent\n ) |>\n arrange(agent)\n#> # A tibble: 119 × 3\n#> agent mean n\n#> <fct> <dbl> <int>\n#> 1 aaron_marquez 118. 2\n#> 2 alexander_drake 144. 1117\n#> 3 allen_her 65 1\n#> 4 anas_el_bashir 99 1\n#> 5 araseli_billy 40 1\n#> 6 arhab_al_islam 35 7\n#> 7 audray_tucker 76.0 38\n#> 8 bernice_baltierra 71.3 35\n#> 9 betzy_rodriguez 84.0 66\n#> 10 brayan_guerrero 37.5 2\n#> # ℹ 109 more rows" }, { - "objectID": "slides/advanced-01-introduction.html#hotel-data", - "href": "slides/advanced-01-introduction.html#hotel-data", - "title": "1 - Introduction", - "section": "Hotel Data ", - "text": "Hotel Data \nWe’ll use data on hotels to predict the cost of a room.\nThe data are in the modeldata package. We’ll sample down the data and refactor some columns:\n\n\n\nlibrary(tidymodels)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#partial-pooling", + "href": "slides/advanced-05-feature-engineering-part-two.html#partial-pooling", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Partial pooling", + "text": "Partial pooling\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage, though." }, { - "objectID": "slides/advanced-01-introduction.html#hotel-date-columns", - "href": "slides/advanced-01-introduction.html#hotel-date-columns", - "title": "1 - Introduction", - "section": "Hotel date columns", - "text": "Hotel date columns\n\nnames(hotel_rates)\n#> [1] \"avg_price_per_room\" \"lead_time\" \n#> [3] \"stays_in_weekend_nights\" \"stays_in_week_nights\" \n#> [5] \"adults\" \"children\" \n#> [7] \"babies\" \"meal\" \n#> [9] \"country\" \"market_segment\" \n#> [11] \"distribution_channel\" \"is_repeated_guest\" \n#> [13] \"previous_cancellations\" \"previous_bookings_not_canceled\"\n#> [15] \"reserved_room_type\" \"assigned_room_type\" \n#> [17] \"booking_changes\" \"agent\" \n#> [19] \"company\" \"days_in_waiting_list\" \n#> [21] \"customer_type\" \"required_car_parking_spaces\" \n#> [23] \"total_of_special_requests\" \"arrival_date_num\" \n#> [25] \"near_christmas\" \"near_new_years\" \n#> [27] \"historical_adr\"" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#partial-pooling-results", + "href": "slides/advanced-05-feature-engineering-part-two.html#partial-pooling-results", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Partial pooling results", + "text": "Partial pooling results" }, { - "objectID": "slides/advanced-01-introduction.html#data-splitting-strategy", - "href": "slides/advanced-01-introduction.html#data-splitting-strategy", - "title": "1 - Introduction", - "section": "Data splitting strategy", - "text": "Data splitting strategy" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#implentations", + "href": "slides/advanced-05-feature-engineering-part-two.html#implentations", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Implentations", + "text": "Implentations\nWe have described this method solely based on analytical calculations (step_lencode()), but you could arrive at similar numbers using a model-based approach by fitting a no-intercept generalized linear model. A hierarchical version would induce partial pooling.\n\nstep_lencode_glm()\nstep_lencode_bayes()\nstep_lencode_mixed()" }, { - "objectID": "slides/advanced-01-introduction.html#data-spending", - "href": "slides/advanced-01-introduction.html#data-spending", - "title": "1 - Introduction", - "section": "Data Spending ", - "text": "Data Spending \nLet’s split the data into a training set (75%) and testing set (25%) using stratification:\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-in-recipes", + "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-in-recipes", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Target encoding in recipes ", + "text": "Target encoding in recipes \n\nrecipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_lencode(\n agent, country, company,\n outcome = vars(\"avg_price_per_room\"), smooth = TRUE,\n ) |>\n prep() |>\n bake(NULL) |>\n select(agent, country, company)\n#> # A tibble: 11,551 × 3\n#> agent country company\n#> <dbl> <dbl> <dbl>\n#> 1 144. 108. 109.\n#> 2 77.2 83.4 109.\n#> 3 61.5 99.9 109.\n#> 4 126. 108. 109.\n#> 5 144. 108. 109.\n#> 6 79.9 73.8 109.\n#> 7 126. 99.9 109.\n#> 8 69.8 108. 109.\n#> 9 126. 99.9 109.\n#> 10 144. 108. 109.\n#> # ℹ 11,541 more rows" }, { - "objectID": "slides/advanced-01-introduction.html#your-turn", - "href": "slides/advanced-01-introduction.html#your-turn", - "title": "1 - Introduction", + "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn-2", + "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn-2", + "title": "5 - Feature engineering: splines, target encoding and dates", "section": "Your turn", - "text": "Your turn\nLet’s take some time and investigate the training data. The outcome is avg_price_per_room.\nAre there any interesting characteristics of the data?\n\n\n\n−+\n10:00" + "text": "Your turn\n\nApply target encoding to the data set, see how it affects different predictors, not just the ones we listed here.\n\nstep_lencode()\nstep_lencode_glm()\nstep_lencode_bayes()\nstep_lencode_mixed()\n\n\n\n\n−+\n03:00" }, - { - "objectID": "slides/advanced-01-introduction.html#our-versions", - "href": "slides/advanced-01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nR version 4.5.1 (2025-06-13), Quarto (1.7.32)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.4.0\n\n\nbroom\n1.0.9\n\n\nCubist\n0.5.0\n\n\ndials\n1.4.2\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.4\n\n\nembed\n1.1.5\n\n\nfinetune\n1.2.1\n\n\nforested\n0.2.0\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nFormula\n1.2-5\n\n\nggplot2\n3.5.2\n\n\nlattice\n0.22-7\n\n\nlightgbm\n4.6.0\n\n\nlme4\n1.1-37\n\n\nmodeldata\n1.5.1\n\n\nparallelly\n1.45.1\n\n\nparsnip\n1.3.3\n\n\nplotmo\n3.6.4\n\n\nplotrix\n3.8-4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nplumber\n1.3.0\n\n\nprobably\n1.1.1\n\n\npurrr\n1.1.0\n\n\nrecipes\n1.3.1\n\n\nrsample\n1.3.1\n\n\nrules\n1.0.2\n\n\nscales\n1.4.0\n\n\nsplines2\n0.5.4\n\n\nstacks\n1.1.1\n\n\ntext2vec\n0.6.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntextrecipes\n1.1.0\n\n\ntibble\n3.3.0\n\n\ntidymodels\n1.3.0\n\n\ntidyr\n1.3.1\n\n\ntune\n2.0.0\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.3.0\n\n\nworkflowsets\n1.1.1\n\n\nyardstick\n1.3.2" + { + "objectID": "slides/advanced-05-feature-engineering-part-two.html#date-time-variables-1", + "href": "slides/advanced-05-feature-engineering-part-two.html#date-time-variables-1", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Date time variables", + "text": "Date time variables\nHow can we represent the date column arrival_date for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model.\n\n\nFEAZ" }, { - "objectID": "index.html", - "href": "index.html", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels offered at posit::conf(2025). The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to Machine Learning in R with tidymodels and Getting More Out of Feature Engineering and Tuning for Machine Learning courses.\nIntroduction to Machine Learning in R with tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nGetting More Out of Feature Engineering and Tuning for Machine Learning will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn-3", + "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn-3", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Your turn", + "text": "Your turn\n\nExplore the arrival_date variable and its relation to avg_price_per_room\nThe lubridate package might provide helpful\n\n\n\n−+\n05:00" }, { - "objectID": "index.html#welcome", - "href": "index.html#welcome", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels offered at posit::conf(2025). The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to Machine Learning in R with tidymodels and Getting More Out of Feature Engineering and Tuning for Machine Learning courses.\nIntroduction to Machine Learning in R with tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nGetting More Out of Feature Engineering and Tuning for Machine Learning will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#arrival_date-date-features", + "href": "slides/advanced-05-feature-engineering-part-two.html#arrival_date-date-features", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "arrival_date date features", + "text": "arrival_date date features\nusing step_date( features = c(\"year\", \"month\", \"dow\", \"decimal\", \"mday\", \"doy\", \"week\", \"semester\", \"quarter\"))\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\narrival_date\nyear\nmonth\ndow\ndecimal\nmday\ndoy\nweek\nsemester\nquarter\n\n\n\n\n2016-08-30\n2016\nAug\nTue\n2016.661\n30\n243\n35\n2\n3\n\n\n2016-10-22\n2016\nOct\nSat\n2016.806\n22\n296\n43\n2\n4\n\n\n2016-12-17\n2016\nDec\nSat\n2016.959\n17\n352\n51\n2\n4\n\n\n2017-02-13\n2017\nFeb\nMon\n2017.118\n13\n44\n7\n1\n1\n\n\n2017-04-05\n2017\nApr\nWed\n2017.258\n5\n95\n14\n1\n2" }, { - "objectID": "index.html#is-this-workshop-for-me", - "href": "index.html#is-this-workshop-for-me", - "title": "Machine learning with tidymodels", - "section": "Is this workshop for me? ", - "text": "Is this workshop for me? \nDepending on your background, one of Introduction to Machine Learning in R with tidymodels or Getting More Out of Feature Engineering and Tuning for Machine Learning might serve you better than the other.\n\nIntroduction to Machine Learning in R with tidymodels\nThis workshop is for you if you:\n\nare comfortable using tidyverse packages to read data into R, transform and reshape data, and make a variety of graphs, and\nhave had some exposure to basic statistical concepts such as linear models, residuals, etc.\n\nIntermediate or expert familiarity with modeling or machine learning is not required. Interested students who have intermediate or expert familiarity with modeling or machine learning may be interested in the Getting More Out of Feature Engineering and Tuning for Machine Learning workshop.\n\n\nGetting More Out of Feature Engineering and Tuning for Machine Learning\nThis workshop is for you if you:\n\nhave the prerequisite skills listed for the Introduction to Machine Learning in R with tidymodels workshops,\nhave used tidymodels packages like recipes, rsample, and parsnip, and\nhave some experience with evaluating statistical models using resampling techniques like v-fold cross-validation or the bootstrap.\n\nParticipants who are new to tidymodels or machine learning will benefit from taking the Introduction to Machine Learning in R with tidymodels workshop before joining this one. Participants who have completed the “Introduction to Machine Learning in R with tidymodels” workshop previously will be well-prepared for this course." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#arrival_date-date-features-1", + "href": "slides/advanced-05-feature-engineering-part-two.html#arrival_date-date-features-1", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "arrival_date date features", + "text": "arrival_date date features\nAdding label = FALSE\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\narrival_date\nyear\nmonth\ndow\ndecimal\nmday\ndoy\nweek\nsemester\nquarter\n\n\n\n\n2016-08-30\n2016\n8\n3\n2016.661\n30\n243\n35\n2\n3\n\n\n2016-10-22\n2016\n10\n7\n2016.806\n22\n296\n43\n2\n4\n\n\n2016-12-17\n2016\n12\n7\n2016.959\n17\n352\n51\n2\n4\n\n\n2017-02-13\n2017\n2\n2\n2017.118\n13\n44\n7\n1\n1\n\n\n2017-04-05\n2017\n4\n4\n2017.258\n5\n95\n14\n1\n2" }, { - "objectID": "index.html#preparation", - "href": "index.html#preparation", - "title": "Machine learning with tidymodels", - "section": "Preparation", - "text": "Preparation\nThe process to set up your computer for either workshop will look the same. Please join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of an IDE, either\n\nRStudio Desktop (RStudio Desktop Open Source License, at least v2024.04.0), available at https://posit.co/download/rstudio-desktop/ or\nPositron (Desktop version, at least 2025.08.0-130), available at https://positron.posit.co/download.html\n\nFor all of the slides, the following R packages can be installed from the R console:\n\n\n# Install the packages for the workshop\npkgs <- \n c(\"almanac\", \"betacal\", \"bonsai\", \"C50\", \"Cubist\", \"desirability2\", \n \"dimRed\", \"embed\", \"extrasteps\", \"finetune\", \"forested\", \n \"igraph\", \"important\", \"irlba\", \"kknn\", \"lightgbm\", \"lme4\", \"mirai\", \n \"parallelly\", \"plumber\", \"probably\", \"ranger\", \"RANN\", \"rpart\", \n \"rpart.plot\", \"RSpectra\", \"rules\", \"splines2\", \"stacks\", \"text2vec\", \n \"textrecipes\", \"tidymodels\", \"uwot\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nAlso, you should make sure that you have installed the newest version of a few packages. To check this, you can run the following code snippet. It will also prompt you to install the required versions if they are not already installed.\n\nrlang::check_installed(\"tidymodels\", version = \"1.4.1\")\nrlang::check_installed(\"embed\", version = \"1.2.0\")\n\nIf you’re a Windows user and encounter an error message during installation noting a missing Rtools installation, install Rtools using the installer linked here." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#other-recipes-steps", + "href": "slides/advanced-05-feature-engineering-part-two.html#other-recipes-steps", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Other recipes steps", + "text": "Other recipes steps\n\nstep_time()\n\nWorks the same as step_date() but for measurements smaller than day: hour, hour12, am/pm, minute, second, decimal_day.\n\nstep_holiday()\n\nAdds indicators for holidays. See timeDate::listHolidays() for supported holidays." }, { - "objectID": "index.html#slides", - "href": "index.html#slides", - "title": "Machine learning with tidymodels", - "section": "Slides", - "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. They are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to Machine Learning in R with tidymodels\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n05: Tuning models\n06: Wrapping up\n\n\n\nGetting More Out of Feature Engineering and Tuning for Machine Learning\n\n01: Introduction\n02: Model optimization by tuning\n03: Grid search via racing\n04: Feature engieering:dummies and embeddings\n05: Feature engineering: splines, target encoding and dates\n06: Postprocessing\n07: Feature selection\n08: Wrapping up\n\n\n\nExtra content (time permitting)\n\nIntro: Using workflowsets\nIntro: Using recipes\nAdvanced: Transit case study (includes stacking)\nAdvanced: Effect encoding\nAdvanced: Model deployment\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn-4", + "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn-4", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Your turn", + "text": "Your turn\n\nApply date steps to the arrival_date variable and try to see if we capture anything about avg_price_per_room\n\n\n\n−+\n03:00" }, { - "objectID": "index.html#code", - "href": "index.html#code", - "title": "Machine learning with tidymodels", - "section": "Code", - "text": "Code\nQuarto files for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#dates-as-numerics", + "href": "slides/advanced-05-feature-engineering-part-two.html#dates-as-numerics", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "dates as numerics", + "text": "dates as numerics" }, { - "objectID": "index.html#past-workshops", - "href": "index.html#past-workshops", - "title": "Machine learning with tidymodels", - "section": "Past workshops", - "text": "Past workshops\n\nEnglish\n\nAugust 2025 at New York Data Science & AI Conference\nAugust 2024 at posit::conf()\nSeptember 2023 at posit::conf()\nJuly 2023 at the New York R Conference\nAugust 2022 in Reykjavik\nJuly 2022 at rstudio::conf()\n\n\n\nSpanish\n\nMarch 2024 at conectaR" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#holidays-as-numerics", + "href": "slides/advanced-05-feature-engineering-part-two.html#holidays-as-numerics", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "holidays as numerics", + "text": "holidays as numerics" }, { - "objectID": "index.html#acknowledgments", - "href": "index.html#acknowledgments", - "title": "Machine learning with tidymodels", - "section": "Acknowledgments", - "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#what-are-the-issues-with-these-features", + "href": "slides/advanced-05-feature-engineering-part-two.html#what-are-the-issues-with-these-features", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "What are the issues with these features?", + "text": "What are the issues with these features?\nThe numeric features make it easy to capture the end or beginning, but harder to do anything more granular.\nThe indicators mostly care about the day itself. No information about the lead-up or aftermath" }, { - "objectID": "index.html#reuse-and-licensing", - "href": "index.html#reuse-and-licensing", - "title": "Machine learning with tidymodels", - "section": "Reuse and licensing", - "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#time-events", + "href": "slides/advanced-05-feature-engineering-part-two.html#time-events", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "time events", + "text": "time events\nUsing extrasteps::step_time_event() and the almanac package, we can create useful time features.\n\nlibrary(almanac)\n\nrule_1 <- weekly() |>\n recur_on_weekdays() |>\n rsetdiff(hol_christmas())\n\nrule_2 <- monthly(since = \"2000-01-01\") |>\n recur_on_interval(3) |>\n recur_on_day_of_month(1)\n\nrule_3 <- yearly(\"1997-06-05\") |>\n recur_on_day_of_week(\"Thursday\") |>\n recur_on_month_of_year(c(\"Jun\", \"July\", \"Aug\"))" }, { - "objectID": "slides/advanced-02-feature-engineering.html#working-with-our-predictors", - "href": "slides/advanced-02-feature-engineering.html#working-with-our-predictors", - "title": "2 - Feature Engineering", - "section": "Working with our predictors", - "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for linear regression).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_time_event", + "href": "slides/advanced-05-feature-engineering-part-two.html#step_time_event", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "step_time_event() ", + "text": "step_time_event() \nCreate a list of rules (last slide) and pass them to the rules argument of extrasteps::step_time_event()\n\nrules <- list(rule_1 = rule_1, rule_2 = rule_2, rule_3 = rule_3)\n\nrecipe(~arrival_date, data = hotel_rates) |>\n step_time_event(arrival_date, rules = rules)\n\n\nFEAZ" }, { - "objectID": "slides/advanced-02-feature-engineering.html#what-is-feature-engineering", - "href": "slides/advanced-02-feature-engineering.html#what-is-feature-engineering", - "title": "2 - Feature Engineering", - "section": "What is feature engineering?", - "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPrincipal component analysis (PCA) feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection (FES) and Applied Machine Learning for Tabular Data (aml4td)." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_time_event-as-numerics", + "href": "slides/advanced-05-feature-engineering-part-two.html#step_time_event-as-numerics", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "step_time_event() as numerics ", + "text": "step_time_event() as numerics" }, { - "objectID": "slides/advanced-02-feature-engineering.html#example-dates", - "href": "slides/advanced-02-feature-engineering.html#example-dates", - "title": "2 - Feature Engineering", - "section": "Example: Dates", - "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#non-indicator-time-events", + "href": "slides/advanced-05-feature-engineering-part-two.html#non-indicator-time-events", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Non-indicator time events", + "text": "Non-indicator time events\nThese features still have the issue that they only attach value to the date itself.\nWe can attach values based on how far we are away from those dates.\n\nstep_date_before()\nstep_date_after()\nstep_date_nearest()" }, { - "objectID": "slides/advanced-02-feature-engineering.html#general-definitions", - "href": "slides/advanced-02-feature-engineering.html#general-definitions", - "title": "2 - Feature Engineering", - "section": "General definitions ", - "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_date_before", + "href": "slides/advanced-05-feature-engineering-part-two.html#step_date_before", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "step_date_before()", + "text": "step_date_before()" }, { - "objectID": "slides/advanced-02-feature-engineering.html#previously---setup", - "href": "slides/advanced-02-feature-engineering.html#previously---setup", - "title": "2 - Feature Engineering", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\n\n# Add another package:\nlibrary(textrecipes)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_date_before---inverse", + "href": "slides/advanced-05-feature-engineering-part-two.html#step_date_before---inverse", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "step_date_before() - inverse", + "text": "step_date_before() - inverse" }, { - "objectID": "slides/advanced-02-feature-engineering.html#previously---data-usage", - "href": "slides/advanced-02-feature-engineering.html#previously---data-usage", - "title": "2 - Feature Engineering", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\n\nWe’ll go from here and create a set of resamples to use for model assessments." + "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_date_after---inverse", + "href": "slides/advanced-05-feature-engineering-part-two.html#step_date_after---inverse", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "step_date_after() - inverse", + "text": "step_date_after() - inverse" }, { - "objectID": "slides/advanced-02-feature-engineering.html#resampling-strategy", - "href": "slides/advanced-02-feature-engineering.html#resampling-strategy", - "title": "2 - Feature Engineering", - "section": "Resampling Strategy", - "text": "Resampling Strategy" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_date_nearest---inverse", + "href": "slides/advanced-05-feature-engineering-part-two.html#step_date_nearest---inverse", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "step_date_nearest() - inverse", + "text": "step_date_nearest() - inverse" }, { - "objectID": "slides/advanced-02-feature-engineering.html#resampling-strategy-1", - "href": "slides/advanced-02-feature-engineering.html#resampling-strategy-1", - "title": "2 - Feature Engineering", - "section": "Resampling Strategy ", - "text": "Resampling Strategy \nWe’ll use simple 10-fold cross-validation (stratified sampling):\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)\nhotel_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [3372/377]> Fold01\n#> 2 <split [3373/376]> Fold02\n#> 3 <split [3373/376]> Fold03\n#> 4 <split [3373/376]> Fold04\n#> 5 <split [3373/376]> Fold05\n#> 6 <split [3374/375]> Fold06\n#> 7 <split [3375/374]> Fold07\n#> 8 <split [3376/373]> Fold08\n#> 9 <split [3376/373]> Fold09\n#> 10 <split [3376/373]> Fold10" + "objectID": "slides/advanced-05-feature-engineering-part-two.html#datetime-features", + "href": "slides/advanced-05-feature-engineering-part-two.html#datetime-features", + "title": "5 - Feature engineering: splines, target encoding and dates", + "section": "Datetime features", + "text": "Datetime features\nAvoid crafting datetime features by hand if at all possible.\nDealing with uneven month lengths, leap days (leap seconds)\nOr tried to define any event that doesn’t land on the same day of the week or date each year.\n\nThe first Sunday after the first full moon on or after the vernal equinox" }, { - "objectID": "slides/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", - "href": "slides/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", - "title": "2 - Feature Engineering", - "section": "Prepare your data for modeling ", - "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of preprocessing and feature engineering steps.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." + "objectID": "slides/advanced-03-racing.html#startup", + "href": "slides/advanced-03-racing.html#startup", + "title": "3 - Racing", + "section": "Startup! ", + "text": "Startup! \n\nlibrary(tidymodels)\nlibrary(finetune)\nlibrary(bonsai)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)" }, { - "objectID": "slides/advanced-02-feature-engineering.html#a-first-recipe", - "href": "slides/advanced-02-feature-engineering.html#a-first-recipe", - "title": "2 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" + "objectID": "slides/advanced-03-racing.html#more-startup", + "href": "slides/advanced-03-racing.html#more-startup", + "title": "3 - Racing", + "section": "More startup! ", + "text": "More startup! \n\n# Load our example data for this section\n\"https://raw.githubusercontent.com/tidymodels/\" |> \n paste0(\"workshops/main/slides/class_data.RData\") |> \n url() |> \n load()\n\nset.seed(429)\nsim_split <- initial_split(class_data, prop = 0.75, strata = class)\nsim_train <- training(sim_split)\nsim_test <- testing(sim_split)\n\nset.seed(523)\nsim_rs <- vfold_cv(sim_train, v = 10, strata = class)" }, { - "objectID": "slides/advanced-02-feature-engineering.html#a-first-recipe-1", - "href": "slides/advanced-02-feature-engineering.html#a-first-recipe-1", - "title": "2 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nsummary(hotel_rec)\n#> # A tibble: 27 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 lead_time <chr [2]> predictor original\n#> 2 stays_in_weekend_nights <chr [2]> predictor original\n#> 3 stays_in_week_nights <chr [2]> predictor original\n#> 4 adults <chr [2]> predictor original\n#> 5 children <chr [2]> predictor original\n#> 6 babies <chr [2]> predictor original\n#> 7 meal <chr [3]> predictor original\n#> 8 country <chr [3]> predictor original\n#> 9 market_segment <chr [3]> predictor original\n#> 10 distribution_channel <chr [3]> predictor original\n#> # ℹ 17 more rows\n\nThe type column contains information on the variables" + "objectID": "slides/advanced-03-racing.html#first-a-shameless-promotion", + "href": "slides/advanced-03-racing.html#first-a-shameless-promotion", + "title": "3 - Racing", + "section": "First, a shameless promotion", + "text": "First, a shameless promotion" }, { - "objectID": "slides/advanced-02-feature-engineering.html#your-turn", - "href": "slides/advanced-02-feature-engineering.html#your-turn", - "title": "2 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\nWhat do you think are in the type vectors for the lead_time and country columns?\n\n\n\n−+\n02:00" + "objectID": "slides/advanced-03-racing.html#making-grid-search-more-efficient", + "href": "slides/advanced-03-racing.html#making-grid-search-more-efficient", + "title": "3 - Racing", + "section": "Making Grid Search More Efficient", + "text": "Making Grid Search More Efficient\nPreviously, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\n\nAlso, for some models, we can fit far fewer models than the number being evaluated.\n\nFor example, with boosted trees, a model with X trees can often predict on candidates with fewer than X trees (i.e., no retraining).\n\nThese strategies can lead to enormous speed-ups." }, { - "objectID": "slides/advanced-02-feature-engineering.html#create-indicator-variables", - "href": "slides/advanced-02-feature-engineering.html#create-indicator-variables", - "title": "2 - Feature Engineering", - "section": "Create indicator variables ", - "text": "Create indicator variables \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns.\nstep_dummy() records the levels of the categorical predictors in the training set." + "objectID": "slides/advanced-03-racing.html#model-racing", + "href": "slides/advanced-03-racing.html#model-racing", + "title": "3 - Racing", + "section": "Model Racing", + "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models, but only for a few resamples.\nDetermine which candidates have a low probability of being selected (cough, cough, tanh activation, cough).\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain).\n\nThis can result in fitting a small number of models.\nIt is not an iterative search; it is an adaptive grid search.\n\nTMwR, TMwR example, AML4TD" }, { - "objectID": "slides/advanced-02-feature-engineering.html#filter-out-constant-columns", - "href": "slides/advanced-02-feature-engineering.html#filter-out-constant-columns", - "title": "2 - Feature Engineering", - "section": "Filter out constant columns ", - "text": "Filter out constant columns \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" + "objectID": "slides/advanced-03-racing.html#discarding-candidates", + "href": "slides/advanced-03-racing.html#discarding-candidates", + "title": "3 - Racing", + "section": "Discarding Candidates", + "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large resampling effect in the results." }, { - "objectID": "slides/advanced-02-feature-engineering.html#normalization", - "href": "slides/advanced-02-feature-engineering.html#normalization", - "title": "2 - Feature Engineering", - "section": "Normalization ", - "text": "Normalization \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." + "objectID": "slides/advanced-03-racing.html#resampling-results-non-racing", + "href": "slides/advanced-03-racing.html#resampling-results-non-racing", + "title": "3 - Racing", + "section": "Resampling Results (Non-Racing)", + "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." }, { - "objectID": "slides/advanced-02-feature-engineering.html#reduce-correlation", - "href": "slides/advanced-02-feature-engineering.html#reduce-correlation", - "title": "2 - Feature Engineering", - "section": "Reduce correlation ", - "text": "Reduce correlation \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." + "objectID": "slides/advanced-03-racing.html#are-candidates-different", + "href": "slides/advanced-03-racing.html#are-candidates-different", + "title": "3 - Racing", + "section": "Are Candidates Different?", + "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval for the difference in the model error is (0.99, 2.8), indicating that candidate number 2 has a smaller error." }, { - "objectID": "slides/advanced-02-feature-engineering.html#other-possible-steps", - "href": "slides/advanced-02-feature-engineering.html#other-possible-steps", - "title": "2 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" + "objectID": "slides/advanced-03-racing.html#evaluating-differences-in-candidates", + "href": "slides/advanced-03-racing.html#evaluating-differences-in-candidates", + "title": "3 - Racing", + "section": "Evaluating Differences in Candidates", + "text": "Evaluating Differences in Candidates\n\n\nWhat if we were to have compared the candidates while we sequentially evaluated each resample?\n👉\n\nOne candidate shows superiority when 5 resamples have been evaluated." }, { - "objectID": "slides/advanced-02-feature-engineering.html#other-possible-steps-1", - "href": "slides/advanced-02-feature-engineering.html#other-possible-steps-1", - "title": "2 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n embed::step_umap(all_numeric_predictors(), outcome = vars(avg_price_per_room))\n\n\nA fancy machine learning supervised dimension reduction technique called UMAP…\n\nNote that this uses the outcome, and it is from an extension package" + "objectID": "slides/advanced-03-racing.html#interim-analysis-of-results", + "href": "slides/advanced-03-racing.html#interim-analysis-of-results", + "title": "3 - Racing", + "section": "Interim Analysis of Results", + "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order (so set the seed).\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." }, { - "objectID": "slides/advanced-02-feature-engineering.html#other-possible-steps-2", - "href": "slides/advanced-02-feature-engineering.html#other-possible-steps-2", - "title": "2 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" + "objectID": "slides/advanced-03-racing.html#boosted-trees-1", + "href": "slides/advanced-03-racing.html#boosted-trees-1", + "title": "3 - Racing", + "section": "Boosted Trees", + "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved, and new samples are predicted using a weighted average of its votes.\n\nWe’ll focus on the popular lightgbm implementation." }, { - "objectID": "slides/advanced-02-feature-engineering.html#your-turn-1", - "href": "slides/advanced-02-feature-engineering.html#your-turn-1", - "title": "2 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\n\nCreate a recipe() for the hotel data to:\n\nuse a Yeo-Johnson (YJ) transformation on lead_time\nconvert factors to indicator variables\nremove zero-variance variables\nadd the spline technique shown above\n\n\n\n\n−+\n03:00" + "objectID": "slides/advanced-03-racing.html#boosted-tree-tuning-parameters", + "href": "slides/advanced-03-racing.html#boosted-tree-tuning-parameters", + "title": "3 - Racing", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands).\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))." }, { - "objectID": "slides/advanced-02-feature-engineering.html#minimal-recipe", - "href": "slides/advanced-02-feature-engineering.html#minimal-recipe", - "title": "2 - Feature Engineering", - "section": "Minimal recipe ", - "text": "Minimal recipe \n\nhotel_indicators <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_YeoJohnson(lead_time) |>\n step_dummy(all_nominal_predictors()) |>\n step_zv(all_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)" + "objectID": "slides/advanced-03-racing.html#boosted-tree-tuning-parameters-1", + "href": "slides/advanced-03-racing.html#boosted-tree-tuning-parameters-1", + "title": "3 - Racing", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nTBH, it is usually not difficult to optimize these models.\n\n\n\nOften, there are multiple candidate tuning parameter regions with very good results.\nFor example: 👉\n\nTo demonstrate, we’ll look at optimizing five of the tuning parameters." }, { - "objectID": "slides/advanced-02-feature-engineering.html#measuring-performance", - "href": "slides/advanced-02-feature-engineering.html#measuring-performance", - "title": "2 - Feature Engineering", - "section": "Measuring Performance ", - "text": "Measuring Performance \nWe’ll compute two measures: mean absolute error and the coefficient of determination (a.k.a \\(R^2\\)).\n\\[\\begin{align}\nMAE &= \\frac{1}{n}\\sum_{i=1}^n |y_i - \\hat{y}_i| \\notag \\\\\nR^2 &= cor(y_i, \\hat{y}_i)^2\n\\end{align}\\]\nThe focus will be on MAE for parameter optimization. We’ll use a metric set to compute these:\n\nreg_metrics <- metric_set(mae, rsq)" + "objectID": "slides/advanced-03-racing.html#boosted-tree-tuning-parameters-2", + "href": "slides/advanced-03-racing.html#boosted-tree-tuning-parameters-2", + "title": "3 - Racing", + "section": "Boosted Tree Tuning Parameters ", + "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\n\nlgbm_spec <-\n boost_tree(\n trees = tune(),\n learn_rate = tune(),\n mtry = tune(),\n min_n = tune(),\n stop_iter = tune()\n ) |>\n set_mode(\"classification\") |>\n # Turn off within-tree parallel processing; it's faster to run \n # the resamples/configurations in parallel\n set_engine(\"lightgbm\", num_threads = 1) \n\n# No preprocessing required:\nlgbm_wflow <- workflow(class ~ ., lgbm_spec)" }, { - "objectID": "slides/advanced-02-feature-engineering.html#using-a-workflow", - "href": "slides/advanced-02-feature-engineering.html#using-a-workflow", - "title": "2 - Feature Engineering", - "section": "Using a workflow ", - "text": "Using a workflow \n\nset.seed(9)\n\nhotel_lm_wflow <-\n workflow() |>\n add_recipe(hotel_indicators) |>\n add_model(linear_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nhotel_lm_res <-\n hotel_lm_wflow |>\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_lm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.6 10 0.214 pre0_mod0_post0\n#> 2 rsq standard 0.884 10 0.00339 pre0_mod0_post0" + "objectID": "slides/advanced-03-racing.html#racing", + "href": "slides/advanced-03-racing.html#racing", + "title": "3 - Racing", + "section": "Racing ", + "text": "Racing \n\nlibrary(finetune)\n\n# Set this to true to demo\nctrl <- control_race(verbose_elim = FALSE)\n\n# Optimizes on the first metric in the set\ncls_mtr <- metric_set(brier_class, roc_auc, sensitivity, specificity)\n\nmirai::daemons(parallel::detectCores() - 1)\n\nset.seed(321)\nlgbm_res <-\n lgbm_wflow |>\n tune_race_anova( # <- very similar syntax to tune_grid()\n resamples = sim_rs,\n # Let's use a larger grid\n grid = 50,\n control = ctrl,\n metrics = cls_mtr\n )" }, { - "objectID": "slides/advanced-02-feature-engineering.html#your-turn-2", - "href": "slides/advanced-02-feature-engineering.html#your-turn-2", - "title": "2 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n−+\n05:00" + "objectID": "slides/advanced-03-racing.html#racing-results", + "href": "slides/advanced-03-racing.html#racing-results", + "title": "3 - Racing", + "section": "Racing Results ", + "text": "Racing Results \n\nshow_best(lgbm_res, metric = \"brier_class\")\n#> # A tibble: 1 × 11\n#> mtry trees min_n learn_rate stop_iter .metric .estimator mean n std_err\n#> <int> <int> <int> <dbl> <int> <chr> <chr> <dbl> <int> <dbl>\n#> 1 9 1836 9 0.00222 6 brier_… binary 0.0379 10 0.00238\n#> # ℹ 1 more variable: .config <chr>\n\n\n\nTimes using 10 cores: sequential: 605s, parallel: 92s, and parallel racing: 50s.\n\n\n\nParallel was 6.6-fold faster, and racing in parallel was 12.3-fold faster." }, { - "objectID": "slides/advanced-02-feature-engineering.html#holdout-predictions", - "href": "slides/advanced-02-feature-engineering.html#holdout-predictions", - "title": "2 - Feature Engineering", - "section": "Holdout predictions ", - "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nlm_cv_pred <- collect_predictions(hotel_lm_res)\nlm_cv_pred |> print(n = 7)\n#> # A tibble: 3,749 × 5\n#> .pred id avg_price_per_room .row .config \n#> <dbl> <chr> <dbl> <int> <chr> \n#> 1 75.1 Fold01 40 20 pre0_mod0_post0\n#> 2 49.3 Fold01 54 28 pre0_mod0_post0\n#> 3 64.9 Fold01 50 45 pre0_mod0_post0\n#> 4 52.8 Fold01 42 49 pre0_mod0_post0\n#> 5 48.6 Fold01 49 61 pre0_mod0_post0\n#> 6 29.8 Fold01 40 66 pre0_mod0_post0\n#> 7 36.9 Fold01 49 88 pre0_mod0_post0\n#> # ℹ 3,742 more rows" + "objectID": "slides/advanced-03-racing.html#racing-results-1", + "href": "slides/advanced-03-racing.html#racing-results-1", + "title": "3 - Racing", + "section": "Racing Results ", + "text": "Racing Results \n\n\nOnly 378 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_res)" }, { - "objectID": "slides/advanced-02-feature-engineering.html#calibration-plot", - "href": "slides/advanced-02-feature-engineering.html#calibration-plot", - "title": "2 - Feature Engineering", - "section": "Calibration Plot ", - "text": "Calibration Plot \n\nlibrary(probably)\n\ncal_plot_regression(hotel_lm_res)" + "objectID": "slides/advanced-03-racing.html#your-turn", + "href": "slides/advanced-03-racing.html#your-turn", + "title": "3 - Racing", + "section": "Your turn", + "text": "Your turn\n\nRun tune_race_anova() with a different seed and/or a different metric.\nDid you get the same or similar results?\n\n\n\n\n−+\n08:00" }, { - "objectID": "slides/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "href": "slides/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "title": "2 - Feature Engineering", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 unique companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor (in the extra materials)" + "objectID": "slides/advanced-01-introduction.html#venue-information", + "href": "slides/advanced-01-introduction.html#venue-information", + "title": "1 - Introduction", + "section": "Venue information", + "text": "Venue information\n\nThere are gender neutral bathrooms located on floor LL2, next to Chicago A\nA meditation/prayer room is located on floor LL2 in Chicago A\nA lactation room is located on floor LL2 in Chicago B" }, { - "objectID": "slides/advanced-02-feature-engineering.html#per-agent-statistics", - "href": "slides/advanced-02-feature-engineering.html#per-agent-statistics", - "title": "2 - Feature Engineering", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "slides/advanced-01-introduction.html#workshop-policies", + "href": "slides/advanced-01-introduction.html#workshop-policies", + "title": "1 - Introduction", + "section": "Workshop policies", + "text": "Workshop policies\n\nPlease review the posit::conf code of conduct, which applies to all workshops: https://posit.co/code-of-conduct\nCoC site has info on how to report a problem (in person, email, phone)\nPlease do not photograph people wearing red lanyards" }, { - "objectID": "slides/advanced-02-feature-engineering.html#collapsing-factor-levels", - "href": "slides/advanced-02-feature-engineering.html#collapsing-factor-levels", - "title": "2 - Feature Engineering", - "section": "Collapsing factor levels ", - "text": "Collapsing factor levels \nThere is a recipe step that will redefine factor levels based on their frequency in the training set:\n\nhotel_other_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_YeoJohnson(lead_time) |>\n step_other(agent, threshold = 0.001) |>\n step_dummy(all_nominal_predictors()) |>\n step_zv(all_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nUsing this code, 34 agents (out of 98) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." + "objectID": "slides/advanced-01-introduction.html#who-are-you", + "href": "slides/advanced-01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do need intermediate familiarity with modeling or ML\nYou have used some tidymodels packages\nYou have some experience with evaluating statistical models using resampling techniques" }, { - "objectID": "slides/advanced-02-feature-engineering.html#does-othering-help", - "href": "slides/advanced-02-feature-engineering.html#does-othering-help", - "title": "2 - Feature Engineering", - "section": "Does othering help? ", - "text": "Does othering help? \n\nhotel_other_wflow <-\n hotel_lm_wflow |>\n update_recipe(hotel_other_rec)\n\nhotel_other_res <-\n hotel_other_wflow |>\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.213 pre0_mod0_post0\n#> 2 rsq standard 0.884 10 0.00341 pre0_mod0_post0\n\nAbout the same MAE and much faster to complete.\nNow let’s look at a more sophisticated tool called effect feature hashing." + "objectID": "slides/advanced-01-introduction.html#who-are-tidymodels", + "href": "slides/advanced-01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\n+ our TA today, Edgar Ruiz!\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "slides/advanced-02-feature-engineering.html#feature-hashing", - "href": "slides/advanced-02-feature-engineering.html#feature-hashing", - "title": "2 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nBetween agent and company, simple dummy variables would create 198 new columns (that are mostly zeros).\nAnother option is to have a binary indicator that combines some levels of these variables.\nFeature hashing (for more see FES, SMLTAR, TMwR, and aml4td):\n\nuses the character values of the levels\nconverts them to integer hash values\nuses the integers to assign them to a specific indicator column." + "objectID": "slides/advanced-01-introduction.html#asking-for-help", + "href": "slides/advanced-01-introduction.html#asking-for-help", + "title": "1 - Introduction", + "section": "Asking for help", + "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" }, { - "objectID": "slides/advanced-02-feature-engineering.html#feature-hashing-1", - "href": "slides/advanced-02-feature-engineering.html#feature-hashing-1", - "title": "2 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nSuppose we want to use 32 indicator variables for agent.\nFor a agent with value “Max_Kuhn”, a hashing function converts it to an integer (say 210397726).\nTo assign it to one of the 32 columns, we would use modular arithmetic to assign it to a column:\n\n# For \"Max_Kuhn\" put a '1' in column: \n210397726 %% 32\n#> [1] 30\n\nHash functions are meant to emulate randomness." + "objectID": "slides/advanced-01-introduction.html#discord", + "href": "slides/advanced-01-introduction.html#discord", + "title": "1 - Introduction", + "section": "Discord ", + "text": "Discord \n\npos.it/conf-event-portal (login)\nClick on “Join Discord, the virtual networking platform!”\nBrowse Channels -> #workshop-feat-eng-tune" }, { - "objectID": "slides/advanced-02-feature-engineering.html#feature-hashing-pros", - "href": "slides/advanced-02-feature-engineering.html#feature-hashing-pros", - "title": "2 - Feature Engineering", - "section": "Feature Hashing Pros", - "text": "Feature Hashing Pros\n\nThe procedure will automatically work on new values of the predictors.\nIt is fast.\n“Signed” hashes add a sign to help avoid aliasing." + "objectID": "slides/advanced-01-introduction.html#section-2", + "href": "slides/advanced-01-introduction.html#section-2", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "slides/advanced-02-feature-engineering.html#feature-hashing-cons", - "href": "slides/advanced-02-feature-engineering.html#feature-hashing-cons", - "title": "2 - Feature Engineering", - "section": "Feature Hashing Cons", - "text": "Feature Hashing Cons\n\nThere is no real logic behind which factor levels are combined.\nWe don’t know how many columns to add (more in the next section).\nSome columns may have all zeros.\nIf a indicator column is important to the model, we can’t easily determine why.\n\n\nThe signed hash make it slightly more possible to differentiate between confounded levels" + "objectID": "slides/advanced-01-introduction.html#section-3", + "href": "slides/advanced-01-introduction.html#section-3", + "title": "1 - Introduction", + "section": "👀", + "text": "👀\n\n\nhttps://workshops.tidymodels.org" }, { - "objectID": "slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes", - "href": "slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes", - "title": "2 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \nThe textrecipes package has a step that can be added to the recipe:\n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) |>\n step_dummy_hash(company) |>\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nhotel_hash_wflow <-\n hotel_lm_wflow |>\n update_recipe(hash_rec)" + "objectID": "slides/advanced-01-introduction.html#tentative-plan-for-this-workshop", + "href": "slides/advanced-01-introduction.html#tentative-plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Tentative plan for this workshop", + "text": "Tentative plan for this workshop\n\nModel optimization by tuning\n\nGrid search\nRacing\n\nFeature engineering with recipes\nPostprocessing\n\nFeature selection" }, { - "objectID": "slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", - "href": "slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", - "title": "2 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \n\nhotel_hash_res <-\n hotel_hash_wflow |>\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_hash_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.239 pre0_mod0_post0\n#> 2 rsq standard 0.884 10 0.00324 pre0_mod0_post0\n\nAbout the same performance but now we can handle new values." + "objectID": "slides/advanced-01-introduction.html#section-4", + "href": "slides/advanced-01-introduction.html#section-4", + "title": "1 - Introduction", + "section": "", + "text": "Introduce yourself to your neighbors 👋" }, { - "objectID": "slides/advanced-02-feature-engineering.html#debugging-a-recipe", - "href": "slides/advanced-02-feature-engineering.html#debugging-a-recipe", - "title": "2 - Feature Engineering", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. hash_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "slides/advanced-01-introduction.html#getting-the-materials", + "href": "slides/advanced-01-introduction.html#getting-the-materials", + "title": "1 - Introduction", + "section": "Getting the materials", + "text": "Getting the materials\n\nIf you are using Posit Cloud:\n Log in to Posit Cloud (free): TODO-ADD-LATER\n\nIf you are working locally:\n# local download\nusethis::use_course(\"tidymodels/workshops\", destdir = \"some_path\")\n\n# or fork via\nusethis::create_from_github(\"tidymodels/workshops\", fork = TRUE)" }, { - "objectID": "slides/advanced-02-feature-engineering.html#example", - "href": "slides/advanced-02-feature-engineering.html#example", - "title": "2 - Feature Engineering", - "section": "Example ", - "text": "Example \n\nhash_rec_fit <- prep(hash_rec)\n\n# Get the transformation coefficient\ntidy(hash_rec_fit, number = 1)\n\n# Get the processed data\nbake(hash_rec_fit, hotel_train |> slice(1:3), contains(\"_agent_\"))" + "objectID": "slides/advanced-01-introduction.html#lets-install-some-packages", + "href": "slides/advanced-01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"almanac\", \"betacal\", \"bonsai\", \"brulee\", \"C50\", \"Cubist\", \"desirability2\", \n \"dimRed\", \"earth\", \"embed\", \"extrasteps\", \"finetune\", \"igraph\", \n \"important\", \"irlba\", \"kknn\", \"lightgbm\", \"lme4\", \"mirai\", \"parallelly\", \n \"plumber\", \"probably\", \"RANN\", \"rpart\", \"RSpectra\", \"rules\", \n \"splines2\", \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \n \"uwot\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nAlso, you should make sure that you have installed the newest version of a few packages. To check this, you can run:\n\nrlang::check_installed(\"tidymodels\", version = \"1.4.1\")\nrlang::check_installed(\"embed\", version = \"1.2.0\")" }, { - "objectID": "slides/advanced-02-feature-engineering.html#more-on-recipes", - "href": "slides/advanced-02-feature-engineering.html#more-on-recipes", - "title": "2 - Feature Engineering", - "section": "More on recipes", - "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." + "objectID": "slides/advanced-01-introduction.html#load-tidymodels", + "href": "slides/advanced-01-introduction.html#load-tidymodels", + "title": "1 - Introduction", + "section": "Load tidymodels ", + "text": "Load tidymodels \nWe’re here to learn more about how to use the more advanced bits of tidymodels for supervised learning. Let’s load the meta-package:\n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.4.1 ──\n#> ✔ broom 1.0.9 ✔ rsample 1.3.1\n#> ✔ dials 1.4.2 ✔ tailor 0.1.0\n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1\n#> ✔ infer 1.0.9 ✔ tune 2.0.0\n#> ✔ modeldata 1.5.1 ✔ workflows 1.3.0\n#> ✔ parsnip 1.3.3 ✔ workflowsets 1.1.1\n#> ✔ purrr 1.1.0 ✔ yardstick 1.3.2\n#> ✔ recipes 1.3.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()" }, { - "objectID": "slides/advanced-04-racing.html#previously---setup", - "href": "slides/advanced-04-racing.html#previously---setup", - "title": "4 - Grid Search via Racing", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/advanced-01-introduction.html#resolve-naming-conflicts", + "href": "slides/advanced-01-introduction.html#resolve-naming-conflicts", + "title": "1 - Introduction", + "section": "Resolve naming conflicts ", + "text": "Resolve naming conflicts \nYou might want to run this function to avoid function name conflicts:\n\n\ntidymodels_prefer()\n\n\nTo get more details, use the quiet = FALSE option." }, { - "objectID": "slides/advanced-04-racing.html#previously---data-usage", - "href": "slides/advanced-04-racing.html#previously---data-usage", - "title": "4 - Grid Search via Racing", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "slides/advanced-01-introduction.html#data-sets", + "href": "slides/advanced-01-introduction.html#data-sets", + "title": "1 - Introduction", + "section": "Data sets", + "text": "Data sets\nFor illustration, we’ll use a few different data sets today:\n\nclass_data: a simulated set of data with a 1:10 class imbalance. Two classes, 20 predictors, and 2,000 data points.\nleaf_data: a real data set to identify plant species from their leaves. Thirty-two levels, 53 predictors, and 1,907 data points.\nhotel_data: a real data set for predicting the average cost per night. Numeric outcome, 27 predictors, and 15,402 data points.\n\n\n\nLet’s get warmed up with the first data set." }, { - "objectID": "slides/advanced-04-racing.html#previously---boosting-model", - "href": "slides/advanced-04-racing.html#previously---boosting-model", - "title": "4 - Grid Search via Racing", - "section": "Previously - Boosting Model ", - "text": "Previously - Boosting Model \n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) |>\n step_dummy_hash(company, num_terms = tune(\"company hash\")) |>\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow |>\n extract_parameter_set_dials() |>\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "slides/advanced-01-introduction.html#imbalanced-data", + "href": "slides/advanced-01-introduction.html#imbalanced-data", + "title": "1 - Introduction", + "section": "Imbalanced data", + "text": "Imbalanced data\nThese data can be loaded from the GitHub repo:\n\n\n\"https://raw.githubusercontent.com/tidymodels/\" |> \n paste0(\"workshops/main/slides/class_data.RData\") |> \n url() |> \n load()\n\n\nThe outcome column is class with levels \"event\" and \"no_event\". Predictors are \"predictor_01\" to \"predictor_30\"." }, { - "objectID": "slides/advanced-04-racing.html#first-a-shameless-promotion", - "href": "slides/advanced-04-racing.html#first-a-shameless-promotion", - "title": "4 - Grid Search via Racing", - "section": "First, a shameless promotion", - "text": "First, a shameless promotion" + "objectID": "slides/advanced-01-introduction.html#your-turn", + "href": "slides/advanced-01-introduction.html#your-turn", + "title": "1 - Introduction", + "section": "Your turn", + "text": "Your turn\nLet’s warm up by taking 8 minutes to explore the data.\n\nWe’ll ask you to tell us something about these data that might be interesting for modeling.\n\n\n\n−+\n08:00" }, { - "objectID": "slides/advanced-04-racing.html#making-grid-search-more-efficient", - "href": "slides/advanced-04-racing.html#making-grid-search-more-efficient", - "title": "4 - Grid Search via Racing", - "section": "Making Grid Search More Efficient", - "text": "Making Grid Search More Efficient\nIn the last section, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\nAlso, for some models, we can fit far fewer models than the number that are being evaluated.\n\nFor boosting, a model with X trees can often predict on candidates with less than X trees.\n\nBoth of these methods can lead to enormous speed-ups." + "objectID": "slides/advanced-01-introduction.html#data-splitting", + "href": "slides/advanced-01-introduction.html#data-splitting", + "title": "1 - Introduction", + "section": "Data splitting ", + "text": "Data splitting \nOne of our first tasks is to split our data into (at a minimum) a training set and a testing set. The rsample package has numerous functions for this, prefixed by initial_. \nLet’s create a 3:1 split of the simulated data and use a stratified random sample (by class):\n\nset.seed(429)\nsim_split <- initial_split(class_data, prop = 0.75, strata = class)\nsim_split\n#> <Training/Testing/Total>\n#> <1499/501/2000>\n\nsim_train <- training(sim_split)\nsim_test <- testing(sim_split)\n\n\nAML4TD, TMwR" }, { - "objectID": "slides/advanced-04-racing.html#model-racing", - "href": "slides/advanced-04-racing.html#model-racing", - "title": "4 - Grid Search via Racing", - "section": "Model Racing", - "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models but only for a few resamples.\nDetermine which candidates have a low probability of being selected.\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain)\n\nThis can result in fitting a small number of models." + "objectID": "slides/advanced-01-introduction.html#data-splitting-1", + "href": "slides/advanced-01-introduction.html#data-splitting-1", + "title": "1 - Introduction", + "section": "Data splitting ", + "text": "Data splitting \n\n\n\nsim_train |> \n ggplot(aes(class)) + \n geom_bar()\n\n\n\n\n\n\n\n\n\n\nsim_test |> \n ggplot(aes(class)) + \n geom_bar()" }, { - "objectID": "slides/advanced-04-racing.html#discarding-candidates", - "href": "slides/advanced-04-racing.html#discarding-candidates", - "title": "4 - Grid Search via Racing", - "section": "Discarding Candidates", - "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large difference between resamples in the results." + "objectID": "slides/advanced-01-introduction.html#resampling", + "href": "slides/advanced-01-introduction.html#resampling", + "title": "1 - Introduction", + "section": "Resampling", + "text": "Resampling\n\n\nWe’ll want to get accurate estimates of model performance.\nLet’s use a resampling method to make multiple versions of our data (using 10-fold cross-validation).\n\nFor large amounts of data, a validation set is also a good alternative.\n\n\n\n\n\n\n\n\nAML4TD, TMwR" }, { - "objectID": "slides/advanced-04-racing.html#resampling-results-non-racing", - "href": "slides/advanced-04-racing.html#resampling-results-non-racing", - "title": "4 - Grid Search via Racing", - "section": "Resampling Results (Non-Racing)", - "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." + "objectID": "slides/advanced-01-introduction.html#resampling-1", + "href": "slides/advanced-01-introduction.html#resampling-1", + "title": "1 - Introduction", + "section": "Resampling ", + "text": "Resampling \n\nset.seed(523)\nsim_rs <- vfold_cv(sim_train, v = 10, strata = class)\nsim_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [1348/151]> Fold01\n#> 2 <split [1349/150]> Fold02\n#> 3 <split [1349/150]> Fold03\n#> 4 <split [1349/150]> Fold04\n#> 5 <split [1349/150]> Fold05\n#> 6 <split [1349/150]> Fold06\n#> 7 <split [1349/150]> Fold07\n#> 8 <split [1349/150]> Fold08\n#> 9 <split [1350/149]> Fold09\n#> 10 <split [1350/149]> Fold10" }, { - "objectID": "slides/advanced-04-racing.html#are-candidates-different", - "href": "slides/advanced-04-racing.html#are-candidates-different", - "title": "4 - Grid Search via Racing", - "section": "Are Candidates Different?", - "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval for the difference in RMSE is (0.99, 2.8), indicating that candidate number 2 has smaller error." + "objectID": "slides/advanced-01-introduction.html#resampled-data-sets", + "href": "slides/advanced-01-introduction.html#resampled-data-sets", + "title": "1 - Introduction", + "section": "Resampled data sets ", + "text": "Resampled data sets \n\nmodel_data_1 <- sim_rs |> get_rsplit(1) |> analysis()\nmodel_data_1 |> count(class)\n#> # A tibble: 2 × 2\n#> class n\n#> <fct> <int>\n#> 1 event 151\n#> 2 no_event 1197\n\nperf_data_1 <- sim_rs |> get_rsplit(1) |> assessment()\nperf_data_1 |> count(class)\n#> # A tibble: 2 × 2\n#> class n\n#> <fct> <int>\n#> 1 event 17\n#> 2 no_event 134" }, { - "objectID": "slides/advanced-04-racing.html#evaluating-differences-in-candidates", - "href": "slides/advanced-04-racing.html#evaluating-differences-in-candidates", - "title": "4 - Grid Search via Racing", - "section": "Evaluating Differences in Candidates", - "text": "Evaluating Differences in Candidates\n\n\nWhat if we were to have compared the candidates while we seqeuntially evaluated each resample?\n👉\n\nOne candidate shows superiority when 4 resamples have been evaluated." + "objectID": "slides/advanced-01-introduction.html#models-via-parsnip", + "href": "slides/advanced-01-introduction.html#models-via-parsnip", + "title": "1 - Introduction", + "section": "Models via parsnip ", + "text": "Models via parsnip \nLet’s fit a simple decision tree to the data:\n\n# Specify what you want\ntree_spec <- decision_tree(mode = \"classification\")\n\n# Then train:\ntree_fit <- tree_spec |> fit(class ~ ., data = model_data_1) \ntree_fit\n#> parsnip model object\n#> \n#> n= 1348 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 1348 151 no_event (0.11201780 0.88798220) \n#> 2) predictor_27< -2.268929 76 11 event (0.85526316 0.14473684) *\n#> 3) predictor_27>=-2.268929 1272 86 no_event (0.06761006 0.93238994) \n#> 6) predictor_29>=2.144976 89 25 event (0.71910112 0.28089888) \n#> 12) predictor_29>=2.599392 45 3 event (0.93333333 0.06666667) *\n#> 13) predictor_29< 2.599392 44 22 event (0.50000000 0.50000000) \n#> 26) predictor_18>=0.2344964 32 11 event (0.65625000 0.34375000) \n#> 52) predictor_03< -0.2257182 16 1 event (0.93750000 0.06250000) *\n#> 53) predictor_03>=-0.2257182 16 6 no_event (0.37500000 0.62500000) *\n#> 27) predictor_18< 0.2344964 12 1 no_event (0.08333333 0.91666667) *\n#> 7) predictor_29< 2.144976 1183 22 no_event (0.01859679 0.98140321) *" }, { - "objectID": "slides/advanced-04-racing.html#interim-analysis-of-results", - "href": "slides/advanced-04-racing.html#interim-analysis-of-results", - "title": "4 - Grid Search via Racing", - "section": "Interim Analysis of Results", - "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order (so set the seed).\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." + "objectID": "slides/advanced-01-introduction.html#predicting", + "href": "slides/advanced-01-introduction.html#predicting", + "title": "1 - Introduction", + "section": "Predicting… ", + "text": "Predicting… \n\npredict(tree_fit, new_data = head(perf_data_1, 4))\n#> # A tibble: 4 × 1\n#> .pred_class\n#> <fct> \n#> 1 event \n#> 2 event \n#> 3 event \n#> 4 no_event\n\npredict(tree_fit, new_data = head(perf_data_1, 4), type = \"prob\")\n#> # A tibble: 4 × 2\n#> .pred_event .pred_no_event\n#> <dbl> <dbl>\n#> 1 0.855 0.145 \n#> 2 0.933 0.0667\n#> 3 0.855 0.145 \n#> 4 0.0833 0.917" }, { - "objectID": "slides/advanced-04-racing.html#racing", - "href": "slides/advanced-04-racing.html#racing", - "title": "4 - Grid Search via Racing", - "section": "Racing ", - "text": "Racing \n\n# Let's use a larger grid\nlgbm_grid <- \n lgbm_param |> \n grid_space_filling(size = 50)\n\nlibrary(finetune)\n\nset.seed(9)\nlgbm_race_res <-\n lgbm_wflow |>\n tune_race_anova(\n resamples = hotel_rs,\n grid = lgbm_grid, \n metrics = reg_metrics\n )\n\nThe syntax and helper functions are extremely similar to those shown for tune_grid()." + "objectID": "slides/advanced-01-introduction.html#augmenting", + "href": "slides/advanced-01-introduction.html#augmenting", + "title": "1 - Introduction", + "section": "Augmenting… ", + "text": "Augmenting… \n\ntree_pred <- augment(tree_fit, new_data = perf_data_1)\ntree_pred |> slice(1:5)\n#> # A tibble: 5 × 34\n#> .pred_class .pred_event .pred_no_event class predictor_01 predictor_02\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl>\n#> 1 event 0.855 0.145 event -1.64 -1.74 \n#> 2 event 0.933 0.0667 event -0.846 0.477 \n#> 3 event 0.855 0.145 event 0.551 0.00571\n#> 4 no_event 0.0833 0.917 event -1.46 -0.335 \n#> 5 event 0.855 0.145 event -0.194 -0.276 \n#> # ℹ 28 more variables: predictor_03 <dbl>, predictor_04 <dbl>,\n#> # predictor_05 <dbl>, predictor_06 <dbl>, predictor_07 <dbl>,\n#> # predictor_08 <dbl>, predictor_09 <dbl>, predictor_10 <dbl>,\n#> # predictor_11 <dbl>, predictor_12 <dbl>, predictor_13 <dbl>,\n#> # predictor_14 <dbl>, predictor_15 <dbl>, predictor_16 <dbl>,\n#> # predictor_17 <dbl>, predictor_18 <dbl>, predictor_19 <dbl>,\n#> # predictor_20 <dbl>, predictor_21 <dbl>, predictor_22 <dbl>, …" }, { - "objectID": "slides/advanced-04-racing.html#racing-results", - "href": "slides/advanced-04-racing.html#racing-results", - "title": "4 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\nshow_best(lgbm_race_res, metric = \"mae\")\n#> # A tibble: 2 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1347 5 0.0655 66 26 mae standard 9.64 10 0.173 pre31_mod34_post0\n#> 2 980 8 0.0429 17 135 mae standard 9.76 10 0.164 pre12_mod25_post0" + "objectID": "slides/advanced-01-introduction.html#performance-metrics", + "href": "slides/advanced-01-introduction.html#performance-metrics", + "title": "1 - Introduction", + "section": "Performance metrics ", + "text": "Performance metrics \nThere are many yardstick metrics* for class predictions and probability estimates.\nLet’s make a collection of metrics and then evaluate our model.\n\ncls_metrics <- metric_set(brier_class, roc_auc, sensitivity, specificity)\ntree_pred |> cls_metrics(truth = class, estimate = .pred_class, .pred_event)\n#> # A tibble: 4 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.647 \n#> 2 specificity binary 0.985 \n#> 3 brier_class binary 0.0429\n#> 4 roc_auc binary 0.899\n\n\n* … and metrics for regression and others.\n\n\nAML4TD" }, { - "objectID": "slides/advanced-04-racing.html#racing-results-1", - "href": "slides/advanced-04-racing.html#racing-results-1", - "title": "4 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\n\nOnly 171 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_race_res) + \n scale_x_continuous(breaks = pretty_breaks())" + "objectID": "slides/advanced-01-introduction.html#recipes-and-workflows", + "href": "slides/advanced-01-introduction.html#recipes-and-workflows", + "title": "1 - Introduction", + "section": "Recipes and workflows ", + "text": "Recipes and workflows \nRecipes are preprocessors that perform sequential operations on the preductors.\n\n\nFor example, to center and scale our predictors:\n\nrec <- \n recipe(class ~ ., data = sim_train) |> \n step_normalize(all_numeric_predictors())\n\n\n\n\nA model, a recipe, and other objects can be added to a workflow to have a single object for the whole modeling sequence:\n\ntree_wflow <- workflow(rec, tree_spec)" }, { - "objectID": "slides/advanced-04-racing.html#your-turn", - "href": "slides/advanced-04-racing.html#your-turn", - "title": "4 - Grid Search via Racing", + "objectID": "slides/advanced-01-introduction.html#your-turn-1", + "href": "slides/advanced-01-introduction.html#your-turn-1", + "title": "1 - Introduction", "section": "Your turn", - "text": "Your turn\n\nRun tune_race_anova() with a different seed.\nDid you get the same or similar results?\n\n\n\n\n−+\n10:00" + "text": "Your turn\nFit a different type of decision tree, this time:\n\nUsing the C5.0 engine\nChange the minimum number of samples required for splitting to 10.\n\n\nDid performance change much?\n\n\n\n\n−+\n05:00" }, { - "objectID": "slides/advanced-06-wrapping-up.html#your-turn", - "href": "slides/advanced-06-wrapping-up.html#your-turn", - "title": "6 - Wrapping up", - "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" + "objectID": "slides/advanced-01-introduction.html#our-versions", + "href": "slides/advanced-01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nR version 4.5.1 (2025-06-13), Quarto (1.7.33)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nalmanac\n1.0.0\n\n\nbetacal\n0.1.0\n\n\nbonsai\n0.4.0\n\n\nbroom\n1.0.9\n\n\nbrulee\n0.6.0.9000\n\n\nC50\n0.2.0\n\n\nCubist\n0.5.0\n\n\nCVST\n0.2-3\n\n\ndesirability2\n0.2.0\n\n\ndials\n1.4.2\n\n\ndimRed\n0.2.7\n\n\ndplyr\n1.1.4\n\n\nDRR\n0.0.4\n\n\nearth\n5.3.4\n\n\nembed\n1.2.0\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nextrasteps\n0.3.0\n\n\nfinetune\n1.2.1\n\n\nforested\n0.2.0\n\n\nFormula\n1.2-5\n\n\nggplot2\n3.5.2\n\n\nigraph\n2.1.4\n\n\nimportant\n0.2.0\n\n\nirlba\n2.3.5.1\n\n\nkernlab\n0.9-33\n\n\nkknn\n1.4.1\n\n\nlattice\n0.22-7\n\n\nlightgbm\n4.6.0\n\n\nlme4\n1.1-37\n\n\nmirai\n2.5.0\n\n\nmodeldata\n1.5.1\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nparallelly\n1.45.1\n\n\nparsnip\n1.3.3\n\n\nplotmo\n3.6.4\n\n\nplotrix\n3.8-4\n\n\nplumber\n1.3.0\n\n\nprobably\n1.1.1\n\n\npurrr\n1.1.0\n\n\nRANN\n2.6.2\n\n\nrecipes\n1.3.1\n\n\nrpart\n4.1.24\n\n\nrsample\n1.3.1\n\n\nRSpectra\n0.16-2\n\n\nrules\n1.0.2\n\n\nscales\n1.4.0\n\n\nsplines2\n0.5.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nstacks\n1.1.1\n\n\ntailor\n0.1.0\n\n\ntext2vec\n0.6.4\n\n\ntextrecipes\n1.1.0\n\n\ntidymodels\n1.4.1\n\n\ntidyr\n1.3.1\n\n\ntune\n2.0.0\n\n\nuwot\n0.2.3\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.3.0\n\n\nworkflowsets\n1.1.1\n\n\nyardstick\n1.3.2" }, { - "objectID": "slides/advanced-06-wrapping-up.html#resources-to-keep-learning", - "href": "slides/advanced-06-wrapping-up.html#resources-to-keep-learning", - "title": "6 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://tidymodels.aml4td.org/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Bluesky, Mastodon and at the tidyverse blog for updates!" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#looking-at-the-predictors", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#looking-at-the-predictors", + "title": "Extras - Recipes", + "section": "Looking at the predictors", + "text": "Looking at the predictors\n\nforested_train\n#> # A tibble: 8,749 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 1997 66 82 10 Tree 12.2 1315 18.4 1.88 24.9 11.8 121 1888 66 -84.9 32.4\n#> 2 No 1997 284 -99 58 Tree 10.3 1236 16.1 -0.26 22.3 9.92 68 1586 80 -85.0 34.1\n#> 3 Yes 2022 130 86 15 Tree 11.8 1194 17.6 1.3 24.1 11.2 61 1753 96 -83.0 33.1\n#> 4 Yes 2021 202 -55 3 Tree 10.7 1235 16.6 0.05 23.0 10.2 72 1682 65 -83.0 33.9\n#> 5 Yes 1995 75 -89 1 Tree 13.8 1256 19.2 3.63 25.5 12.9 57 1796 88 -83.4 31.3\n#> 6 No 1995 110 -53 5 Tree 12.4 1236 18.6 2.53 24.8 12.4 102 1835 51 -83.9 32.2\n#> 7 Yes 2022 111 73 12 Tree 11.5 1168 17.4 1 24.0 10.9 67 1772 84 -82.2 33.6\n#> 8 Yes 1997 230 96 14 Tree 9.98 1373 15.4 -1.35 21.8 9.03 46 1552 68 -85.3 34.8\n#> 9 Yes 2002 160 -88 13 Tree 11.1 1219 16.9 0.07 23.6 10.2 53 1731 95 -83.2 33.6\n#> 10 Yes 2020 39 9 6 Tree 13.9 1237 19.2 3.25 25.6 12.8 58 1812 86 -82.0 31.7\n#> # ℹ 8,739 more rows\n#> # ℹ 2 more variables: land_type <fct>, county <fct>" }, { - "objectID": "slides/extras-effect-encodings.html#previously---setup", - "href": "slides/extras-effect-encodings.html#previously---setup", - "title": "Extras - Effect Encodings", - "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#working-with-other-models", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#working-with-other-models", + "title": "Extras - Recipes", + "section": "Working with other models", + "text": "Working with other models\nSome models can’t handle non-numeric data\n\nLinear Regression\nK Nearest Neighbors\n\n\n\nSome models struggle if numeric predictors aren’t scaled\n\nK Nearest Neighbors\nAnything using gradient descent" }, { - "objectID": "slides/extras-effect-encodings.html#previously---data-usage", - "href": "slides/extras-effect-encodings.html#previously---data-usage", - "title": "Extras - Effect Encodings", - "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#types-of-needed-preprocessing", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#types-of-needed-preprocessing", + "title": "Extras - Recipes", + "section": "Types of needed preprocessing", + "text": "Types of needed preprocessing\n\nDo qualitative predictors require a numeric encoding?\nShould columns with a single unique value be removed?\nDoes the model struggle with missing data?\nDoes the model struggle with correlated predictors?\nShould predictors be centered and scaled?\nIs it helpful to transform predictors to be more symmetric?\n\n\nhttps://www.tmwr.org/pre-proc-table.html" }, { - "objectID": "slides/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "href": "slides/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "title": "Extras - Effect Encodings", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-preprocessing", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-preprocessing", + "title": "Extras - Recipes", + "section": "Two types of preprocessing", + "text": "Two types of preprocessing" }, { - "objectID": "slides/extras-effect-encodings.html#per-agent-statistics", - "href": "slides/extras-effect-encodings.html#per-agent-statistics", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-preprocessing-1", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-preprocessing-1", + "title": "Extras - Recipes", + "section": "Two types of preprocessing", + "text": "Two types of preprocessing" }, { - "objectID": "slides/extras-effect-encodings.html#what-is-an-effect-encoding", - "href": "slides/extras-effect-encodings.html#what-is-an-effect-encoding", - "title": "Extras - Effect Encodings", - "section": "What is an effect encoding?", - "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <fct> <int>\n#> 1 52.7 cynthia_worsley 1\n#> 2 51.8 carlos_bryant 2\n#> 3 53.8 lance_hitchcock 3\n#> 4 51.8 lance_hitchcock 4\n#> 5 46.8 cynthia_worsley 5\n#> 6 54.7 charles_najera 6\n#> 7 46.8 cynthia_worsley 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <dbl> <int>\n#> 1 52.7 88.5 1\n#> 2 51.8 89.5 2\n#> 3 53.8 79.8 3\n#> 4 51.8 79.8 4\n#> 5 46.8 88.5 5\n#> 6 54.7 109. 6\n#> 7 46.8 88.5 7\n\n\nThe agent column is replaced with an estimate of the ADR." + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#general-definitions", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#general-definitions", + "title": "Extras - Recipes", + "section": "General definitions", + "text": "General definitions\n\nData preprocessing is what you do to make your model successful.\nFeature engineering is what you do to the original predictors to make the model do the least work to perform great." }, { - "objectID": "slides/extras-effect-encodings.html#per-agent-statistics-again", - "href": "slides/extras-effect-encodings.html#per-agent-statistics-again", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics again", - "text": "Per-agent statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these means use partial pooling.\nPooling borrows strength across agents and shrinks extreme values towards the mean for agents with very few transations\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for agents with fewer reservations by shrinking the estimate to the overall ADR mean" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#working-with-dates", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#working-with-dates", + "title": "Extras - Recipes", + "section": "Working with dates", + "text": "Working with dates\nDatetime variables are automatically converted to an integer if given as a raw predictor. To avoid this, it can be re-encoded as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nLeap year\nIndicators for holidays" }, { - "objectID": "slides/extras-effect-encodings.html#partial-pooling", - "href": "slides/extras-effect-encodings.html#partial-pooling", - "title": "Extras - Effect Encodings", - "section": "Partial pooling", - "text": "Partial pooling" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-transformations", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-transformations", + "title": "Extras - Recipes", + "section": "Two types of transformations", + "text": "Two types of transformations\n\n\n\nStatic\n\nSquare root, log, inverse\nDummies for known levels\nDate time extractions\n\n\nTrained\n\nCentering & scaling\nImputation\nPCA\nAnything for unknown factor levels\n\n\n\nTrained methods need to calculate sufficient information to be applied again." }, { - "objectID": "slides/extras-effect-encodings.html#agent-effects", - "href": "slides/extras-effect-encodings.html#agent-effects", - "title": "Extras - Effect Encodings", - "section": "Agent effects ", - "text": "Agent effects \n\nlibrary(embed)\n\nhotel_effect_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_YeoJohnson(lead_time) |>\n step_lencode_mixed(agent, company, outcome = vars(avg_price_per_room)) |>\n step_dummy(all_nominal_predictors()) |>\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#the-recipes-package", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#the-recipes-package", + "title": "Extras - Recipes", + "section": "The recipes package", + "text": "The recipes package\n\n\nModular + extensible\nWorks well with pipes ,|> and %>%\nDeferred evaluation\nIsolates test data from training data\nCan do things formulas can’t" }, { - "objectID": "slides/extras-effect-encodings.html#effect-encoding-results", - "href": "slides/extras-effect-encodings.html#effect-encoding-results", - "title": "Extras - Effect Encodings", - "section": "Effect encoding results ", - "text": "Effect encoding results \n\nhotel_effect_wflow <-\n workflow() |>\n add_model(linear_reg()) |> \n update_recipe(hotel_effect_rec)\n\nreg_metrics <- metric_set(mae, rsq)\n\nhotel_effect_res <-\n hotel_effect_wflow |>\n fit_resamples(hotel_rs, metrics = reg_metrics)\n\ncollect_metrics(hotel_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.8 10 0.189 pre0_mod0_post0\n#> 2 rsq standard 0.870 10 0.00357 pre0_mod0_post0\n\nSlightly worse but it can handle new agents (if they occur)." + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())" }, { - "objectID": "slides/extras-vetiver.html#deploying-a-model", - "href": "slides/extras-vetiver.html#deploying-a-model", - "title": "Extras - Model deployment", - "section": "Deploying a model ", - "text": "Deploying a model \nWe have a decision tree, tree_fit, to model whether or not a plot of land in Washington is forested or not.\nHow do we use our model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"forested\")\nv\n#> \n#> ── forested ─ <bundled_workflow> model for deployment \n#> A rpart classification modeling workflow using 19 features\n\nLearn more at https://vetiver.rstudio.com" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-1", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-1", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())\n\n\nStart by calling recipe() to denote the data source and variables used." }, { - "objectID": "slides/extras-vetiver.html#deploy-your-model-1", - "href": "slides/extras-vetiver.html#deploy-your-model-1", - "title": "Extras - Model deployment", - "section": "Deploy your model ", - "text": "Deploy your model \nHow do we use our model in production?\n\nlibrary(plumber)\npr() |>\n vetiver_api(v)\n#> # Plumber router with 4 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Users/hannah/Library/R/arm64/4.5/library/vetiver\n#> ├──/metadata (GET)\n#> ├──/ping (GET)\n#> ├──/predict (POST)\n#> └──/prototype (GET)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-2", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-2", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())\n\n\nSpecify what actions to take by adding step_*()s." }, { - "objectID": "slides/extras-vetiver.html#your-turn", - "href": "slides/extras-vetiver.html#your-turn", - "title": "Extras - Model deployment", - "section": "Your turn", - "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n−+\n05:00" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-3", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-3", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>   step_normalize(all_numeric_predictors())\n\n\nUse {tidyselect} and recipes-specific selectors to denote affected variables." }, { - "objectID": "slides/intro-02-data-budget.html#data-on-forests-in-georgia", - "href": "slides/intro-02-data-budget.html#data-on-forests-in-georgia", - "title": "2 - Your data budget", - "section": "Data on forests in Georgia", - "text": "Data on forests in Georgia\n\n\n\nThe U.S. Forest Service maintains ML models to predict whether a plot of land is “forested.”\nThis classification is important for all sorts of research, legislation, and land management purposes.\nPlots are typically remeasured every 10 years and this dataset contains the most recent measurement per plot.\nType ?forested_ga to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/251793/forest-mountain" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe", + "title": "Extras - Recipes", + "section": "Using a recipe", + "text": "Using a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>   step_normalize(all_numeric_predictors())\n\n\nSave the recipe we like so that we can use it in various places, e.g., with different models." }, { - "objectID": "slides/intro-02-data-budget.html#data-on-forests-in-georgia-1", - "href": "slides/intro-02-data-budget.html#data-on-forests-in-georgia-1", - "title": "2 - Your data budget", - "section": "Data on forests in Georgia", - "text": "Data on forests in Georgia\n\n\n\nN = 10937 plots of land, one from each of 10937 6000-acre hexagons in Georgia.\nA nominal outcome, forested, with levels \"Yes\" and \"No\", measured “on-the-ground.”\n18 remotely-sensed and easily-accessible predictors:\n\nnumeric variables based on weather and topography.\nnominal variables based on classifications from other governmental orgs.\n\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/67614/forest\n\n\n\nThose nominal variables are classifications similar to “forested” but from other agencies. e.g. land_type is from the European Space Agency, and is a remotely-sensed 3-class distribution based on predictions for how the land is used. (The data also includes a county variable, which is not itself from a model.)" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe-with-workflows", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe-with-workflows", + "title": "Extras - Recipes", + "section": "Using a recipe with workflows", + "text": "Using a recipe with workflows\nRecipes are typically combined with a model in a workflow() object:\n\n\nforested_wflow <- workflow() |>\n  add_recipe(forested_rec) |>\n  add_model(linear_reg())" + }, + { + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#recipes-are-estimated", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#recipes-are-estimated", + "title": "Extras - Recipes", + "section": "Recipes are estimated", + "text": "Recipes are estimated\nEvery preprocessing step in a recipe that involved calculations uses the training set. For example:\n\nLevels of a factor\nDetermination of zero-variance\nNormalization\nFeature extraction\n\nOnce a recipe is added to a workflow, this occurs when fit() is called." }, { - "objectID": "slides/intro-02-data-budget.html#checklist-for-predictors", - "href": "slides/intro-02-data-budget.html#checklist-for-predictors", - "title": "2 - Your data budget", - "section": "Checklist for predictors", - "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?\n\n\n\nre: ethics – what issues might arise from releasing the true lat and lon? In reality, these lat and lon are slightly jittered to help ensure trust with landowners who allow surveyers to come take measurements." + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#debugging-a-recipe", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#debugging-a-recipe", + "title": "Extras - Recipes", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. forested_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4.\n\n\n\n\nAnother function, bake(), is analogous to predict(), and gives you the processed data back." }, { - "objectID": "slides/intro-02-data-budget.html#data-on-forests-in-georgia-2", - "href": "slides/intro-02-data-budget.html#data-on-forests-in-georgia-2", - "title": "2 - Your data budget", - "section": "Data on forests in Georgia", - "text": "Data on forests in Georgia\n\nlibrary(tidymodels)\nlibrary(forested)\n\nforested_ga\n#> # A tibble: 10,937 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 2007 14 0 0 No tree 13.9\n#> 2 Yes 2007 66 -53 10 Tree 13.8\n#> 3 Yes 2006 59 -82 6 No tree 13.5\n#> 4 Yes 2007 116 -78 20 Tree 12.3\n#> 5 Yes 2006 283 63 13 Tree 10.0\n#> 6 Yes 2007 250 63 14 Tree 10.8\n#> 7 Yes 2007 58 31 1 No tree 13.8\n#> 8 Yes 2023 140 56 11 Tree 12.2\n#> 9 Yes 2024 118 72 17 Tree 12.2\n#> 10 Yes 2024 217 -46 13 Tree 12.4\n#> # ℹ 10,927 more rows\n#> # ℹ 12 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>, county <fct>" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#your-turn", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#your-turn", + "title": "Extras - Recipes", + "section": "Your turn", + "text": "Your turn\n\n\nTake the recipe and prep() then bake() it to see what the resulting data set looks like.\nTry removing steps to see how the result changes.\n\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-02-data-budget.html#data-splitting-and-spending", - "href": "slides/intro-02-data-budget.html#data-splitting-and-spending", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#printing-a-recipe", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#printing-a-recipe", + "title": "Extras - Recipes", + "section": "Printing a recipe", + "text": "Printing a recipe\n\nforested_rec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Operations\n#> • Dummy variables from: all_nominal_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Log transformation on: canopy_cover\n#> • Centering and scaling for: all_numeric_predictors()" }, { - "objectID": "slides/intro-02-data-budget.html#data-splitting-and-spending-1", - "href": "slides/intro-02-data-budget.html#data-splitting-and-spending-1", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#prepping-a-recipe", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#prepping-a-recipe", + "title": "Extras - Recipes", + "section": "Prepping a recipe", + "text": "Prepping a recipe\n\nprep(forested_rec)\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Training information\n#> Training data contained 8749 data points and no incomplete rows.\n#> \n#> ── Operations\n#> • Dummy variables from: tree_no_tree, land_type, county | Trained\n#> • Zero variance filter removed: <none> | Trained\n#> • Log transformation on: canopy_cover | Trained\n#> • Centering and scaling for: year elevation, ... | Trained" }, { - "objectID": "slides/intro-02-data-budget.html#data-splitting-and-spending-2", - "href": "slides/intro-02-data-budget.html#data-splitting-and-spending-2", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#baking-a-recipe", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#baking-a-recipe", + "title": "Extras - Recipes", + "section": "Baking a recipe", + "text": "Baking a recipe\n\nprep(forested_rec) |>\n bake(new_data = forested_train)\n#> # A tibble: 8,749 × 177\n#> year elevation eastness roughness dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat forested\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> \n#> 1 -1.15 -0.521 1.13 0.0463 -0.232 0.178 0.172 -0.107 0.338 -0.00909 3.10 1.01 0.407 -1.44 -0.0184 Yes \n#> 2 -1.15 1.10 -1.50 3.91 -1.39 -0.568 -1.31 -1.33 -1.28 -1.28 0.0408 -0.941 0.652 -1.51 1.36 No \n#> 3 0.892 -0.0440 1.19 0.449 -0.422 -0.965 -0.305 -0.438 -0.170 -0.438 -0.363 0.140 0.884 0.331 0.556 Yes \n#> 4 0.810 0.493 -0.858 -0.518 -1.11 -0.577 -1.01 -1.15 -0.878 -1.10 0.272 -0.320 0.388 0.331 1.26 Yes \n#> 5 -1.31 -0.454 -1.35 -0.679 0.748 -0.379 0.701 0.893 0.683 0.702 -0.594 0.418 0.773 -0.0643 -0.992 Yes \n#> 6 -1.31 -0.193 -0.829 -0.357 -0.0546 -0.568 0.331 0.264 0.281 0.366 2.00 0.670 0.0802 -0.468 -0.237 No \n#> 7 0.892 -0.186 1.00 0.207 -0.631 -1.21 -0.444 -0.609 -0.264 -0.619 -0.0169 0.262 0.714 1.01 0.929 Yes \n#> 8 -1.15 0.702 1.34 0.369 -1.56 0.726 -1.77 -1.95 -1.60 -1.87 -1.23 -1.16 0.445 -1.77 2.03 Yes \n#> 9 -0.740 0.180 -1.34 0.288 -0.858 -0.729 -0.801 -1.14 -0.515 -1.08 -0.825 -0.00273 0.870 0.122 0.988 Yes \n#> 10 0.729 -0.722 0.0719 -0.276 0.816 -0.559 0.708 0.676 0.745 0.635 -0.536 0.521 0.744 1.22 -0.653 Yes \n#> # ℹ 8,739 more rows\n#> # ℹ 161 more variables: tree_no_tree_No.tree <dbl>, land_type_Non.tree.vegetation <dbl>, land_type_Tree <dbl>, county_Atkinson <dbl>, county_Bacon <dbl>, county_Baker <dbl>, county_Baldwin <dbl>,\n#> # county_Banks <dbl>, county_Barrow <dbl>, county_Bartow <dbl>, county_Ben.Hill <dbl>, county_Berrien <dbl>, county_Bibb <dbl>, county_Bleckley <dbl>, county_Brantley <dbl>, county_Brooks <dbl>,\n#> # county_Bryan <dbl>, county_Bulloch <dbl>, county_Burke <dbl>, county_Butts <dbl>, county_Calhoun <dbl>, county_Camden <dbl>, county_Candler <dbl>, county_Carroll <dbl>, county_Catoosa <dbl>,\n#> # county_Charlton <dbl>, county_Chatham <dbl>, county_Chattahoochee <dbl>, county_Chattooga <dbl>, county_Cherokee <dbl>, county_Clarke <dbl>, county_Clay <dbl>, county_Clayton <dbl>,\n#> # county_Clinch <dbl>, county_Cobb <dbl>, county_Coffee <dbl>, county_Colquitt <dbl>, county_Columbia <dbl>, county_Cook <dbl>, county_Coweta <dbl>, county_Crawford <dbl>, county_Crisp <dbl>,\n#> # county_Dade <dbl>, county_Dawson <dbl>, county_Decatur <dbl>, county_DeKalb <dbl>, county_Dodge <dbl>, county_Dooly <dbl>, county_Dougherty <dbl>, county_Douglas <dbl>, county_Early <dbl>, …" }, { - "objectID": "slides/intro-02-data-budget.html#your-turn", - "href": "slides/intro-02-data-budget.html#your-turn", - "title": "2 - Your data budget", + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\nOnce a recipe as been estimated, there are various bits of information saved in it.\n\nThe tidy() function can be used to get specific results from the recipe." + }, + { + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#your-turn-1", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#your-turn-1", + "title": "Extras - Recipes", "section": "Your turn", - "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" + "text": "Your turn\n\nTake a prepped recipe and use the tidy() function on it.\nUse the number argument to inspect different steps.\n\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-02-data-budget.html#the-initial-split", - "href": "slides/intro-02-data-budget.html#the-initial-split", - "title": "2 - Your data budget", - "section": "The initial split ", - "text": "The initial split \n\nset.seed(123)\nforested_split <- initial_split(forested_ga)\nforested_split\n#> <Training/Testing/Total>\n#> <8202/2735/10937>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe-1", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe-1", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\n\nprep(forested_rec) |>\n tidy()\n#> # A tibble: 4 × 6\n#> number operation type trained skip id \n#> <int> <chr> <chr> <lgl> <lgl> <chr> \n#> 1 1 step dummy TRUE FALSE dummy_hIEnQ \n#> 2 2 step zv TRUE FALSE zv_ZrQBx \n#> 3 3 step log TRUE FALSE log_6es7X \n#> 4 4 step normalize TRUE FALSE normalize_XsIxb" }, { - "objectID": "slides/intro-02-data-budget.html#what-is-set.seed", - "href": "slides/intro-02-data-budget.html#what-is-set.seed", - "title": "2 - Your data budget", - "section": "What is set.seed()?", - "text": "What is set.seed()?\nTo create that split of the data, R generates “pseudo-random” numbers: while they are made to behave like random numbers, their generation is deterministic given a “seed”.\nThis allows us to reproduce results by setting that seed.\nWhich seed you pick doesn’t matter, as long as you don’t try a bunch of seeds and pick the one that gives you the best performance." + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe-2", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe-2", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\n\nprep(forested_rec) |>\n tidy(number = 1)\n#> # A tibble: 161 × 3\n#> terms columns id \n#> <chr> <chr> <chr> \n#> 1 tree_no_tree No tree dummy_hIEnQ\n#> 2 land_type Non-tree vegetation dummy_hIEnQ\n#> 3 land_type Tree dummy_hIEnQ\n#> 4 county Atkinson dummy_hIEnQ\n#> 5 county Bacon dummy_hIEnQ\n#> 6 county Baker dummy_hIEnQ\n#> 7 county Baldwin dummy_hIEnQ\n#> 8 county Banks dummy_hIEnQ\n#> 9 county Barrow dummy_hIEnQ\n#> 10 county Bartow dummy_hIEnQ\n#> # ℹ 151 more rows" }, { - "objectID": "slides/intro-02-data-budget.html#accessing-the-data", - "href": "slides/intro-02-data-budget.html#accessing-the-data", - "title": "2 - Your data budget", - "section": "Accessing the data ", - "text": "Accessing the data \n\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe-in-tidymodels", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe-in-tidymodels", + "title": "Extras - Recipes", + "section": "Using a recipe in tidymodels", + "text": "Using a recipe in tidymodels\nThe recommended way to use a recipe in tidymodels is to use it as part of a workflow().\n\nforested_wflow <- workflow() |> \n add_recipe(forested_rec) |> \n add_model(linear_reg())\n\nWhen used in this way, you don’t need to worry about prep() and bake() as it is handled for you." }, { - "objectID": "slides/intro-02-data-budget.html#the-training-set", - "href": "slides/intro-02-data-budget.html#the-training-set", - "title": "2 - Your data budget", - "section": "The training set", - "text": "The training set\n\nforested_train\n#> # A tibble: 8,202 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 1997 66 82 10 Tree 12.2 \n#> 2 No 1997 284 -99 58 Tree 10.3 \n#> 3 Yes 2022 130 86 15 Tree 11.8 \n#> 4 Yes 2021 202 -55 3 Tree 10.7 \n#> 5 Yes 1995 75 -89 1 Tree 13.8 \n#> 6 No 1995 110 -53 5 Tree 12.4 \n#> 7 Yes 2022 111 73 12 Tree 11.5 \n#> 8 Yes 1997 230 96 14 Tree 9.98\n#> 9 Yes 2002 160 -88 13 Tree 11.1 \n#> 10 Yes 2020 39 9 6 Tree 13.9 \n#> # ℹ 8,192 more rows\n#> # ℹ 12 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>, county <fct>" + "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#more-information", + "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#more-information", + "title": "Extras - Recipes", + "section": "More information", + "text": "More information\n\nhttps://recipes.tidymodels.org/\nhttps://www.tmwr.org/recipes.html" }, { - "objectID": "slides/intro-02-data-budget.html#the-test-set", - "href": "slides/intro-02-data-budget.html#the-test-set", - "title": "2 - Your data budget", - "section": "The test set ", - "text": "The test set \n🙈\n\nThere are 2735 rows and 19 columns in the test set." + "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#tuning-parameters", + "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#tuning-parameters", + "title": "5 - Tuning models", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "slides/intro-02-data-budget.html#your-turn-1", - "href": "slides/intro-02-data-budget.html#your-turn-1", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\nWe recommend using the .qmd files in the classwork/ folder for code exercises. They set you up with the code from the slides.\n\n\n\n−+\n05:00" + "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#optimize-tuning-parameters", + "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#optimize-tuning-parameters", + "title": "5 - Tuning models", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "slides/intro-02-data-budget.html#data-splitting-and-spending-3", - "href": "slides/intro-02-data-budget.html#data-splitting-and-spending-3", - "title": "2 - Your data budget", - "section": "Data splitting and spending ", - "text": "Data splitting and spending \n\nset.seed(123)\nforested_split <- initial_split(forested_ga, prop = 0.8)\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)\n\nnrow(forested_train)\n#> [1] 8749\nnrow(forested_test)\n#> [1] 2188" + "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#optimize-tuning-parameters-1", + "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#optimize-tuning-parameters-1", + "title": "5 - Tuning models", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "slides/intro-02-data-budget.html#your-turn-2", - "href": "slides/intro-02-data-budget.html#your-turn-2", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nExplore the forested_train data on your own!\n\nWhat’s the distribution of the outcome, forested?\nWhat’s the distribution of numeric variables like precip_annual?\nHow does the distribution of forested differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" + "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#specifying-tuning-parameters", + "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#specifying-tuning-parameters", + "title": "5 - Tuning models", + "section": "Specifying tuning parameters", + "text": "Specifying tuning parameters\nLet’s take our previous random forest workflow and tag for tuning the minimum number of data points in each node:\n\nrf_spec <- rand_forest(min_n = tune()) |> \n set_mode(\"classification\")\n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" }, { - "objectID": "slides/intro-02-data-budget.html#section-1", - "href": "slides/intro-02-data-budget.html#section-1", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = forested)) +\n geom_bar()" + "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#try-out-multiple-values", + "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#try-out-multiple-values", + "title": "5 - Tuning models", + "section": "Try out multiple values", + "text": "Try out multiple values\ntune_grid() works similar to fit_resamples() but covers multiple parameter values:\n\nset.seed(22)\nrf_res <- tune_grid(\n rf_wflow,\n forested_folds,\n grid = 5\n)" }, { - "objectID": "slides/intro-02-data-budget.html#section-2", - "href": "slides/intro-02-data-budget.html#section-2", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = forested, fill = tree_no_tree)) +\n geom_bar()" + "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#compare-results", + "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#compare-results", + "title": "5 - Tuning models", + "section": "Compare results", + "text": "Compare results\nInspecting results and selecting the best-performing hyperparameter(s):\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 40 roc_auc binary 0.762 10 0.0107 Preprocessor1_Model1\n#> 2 31 roc_auc binary 0.761 10 0.0101 Preprocessor1_Model2\n#> 3 22 roc_auc binary 0.760 10 0.0103 Preprocessor1_Model5\n#> 4 12 roc_auc binary 0.758 10 0.0101 Preprocessor1_Model4\n#> 5 3 roc_auc binary 0.755 10 0.00969 Preprocessor1_Model3\n\nbest_parameter <- select_best(rf_res)\nbest_parameter\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 40 Preprocessor1_Model1\n\ncollect_metrics() and autoplot() are also available." }, { - "objectID": "slides/intro-02-data-budget.html#section-3", - "href": "slides/intro-02-data-budget.html#section-3", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"identity\", alpha = .7)" + "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#the-final-fit", + "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#the-final-fit", + "title": "5 - Tuning models", + "section": "The final fit", + "text": "The final fit\n\nrf_wflow <- finalize_workflow(rf_wflow, best_parameter)\n\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.764 Preprocessor1_Model1\n#> 2 roc_auc binary 0.764 Preprocessor1_Model1\n#> 3 brier_class binary 0.161 Preprocessor1_Model1" }, { - "objectID": "slides/intro-02-data-budget.html#section-4", - "href": "slides/intro-02-data-budget.html#section-4", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"fill\")" + "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#your-turn", + "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#your-turn", + "title": "5 - Tuning models", + "section": "Your turn", + "text": "Your turn\n\nModify your model workflow to tune one or more parameters.\nUse grid search to find the best parameter(s).\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-02-data-budget.html#section-5", - "href": "slides/intro-02-data-budget.html#section-5", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = lon, y = lat, col = forested)) +\n geom_point()" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets\nbrulee for regression using torch" }, { - "objectID": "slides/intro-02-data-budget.html#the-whole-game---status-update", - "href": "slides/intro-02-data-budget.html#the-whole-game---status-update", - "title": "2 - Your data budget", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "slides/intro-04-evaluating-models.html#looking-at-predictions", - "href": "slides/intro-04-evaluating-models.html#looking-at-predictions", - "title": "4 - Evaluating models", - "section": "Looking at predictions", - "text": "Looking at predictions\n\naugment(forested_fit, new_data = forested_train)\n#> # A tibble: 8,749 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness roughness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 0.931 0.0690 Yes 1997 66 82 10\n#> 2 Yes 0.983 0.0172 No 1997 284 -99 58\n#> 3 Yes 0.960 0.0401 Yes 2022 130 86 15\n#> 4 Yes 0.870 0.130 Yes 2021 202 -55 3\n#> 5 Yes 0.823 0.177 Yes 1995 75 -89 1\n#> 6 Yes 0.758 0.242 No 1995 110 -53 5\n#> 7 Yes 0.823 0.177 Yes 2022 111 73 12\n#> 8 No 0.467 0.533 Yes 1997 230 96 14\n#> 9 Yes 0.983 0.0172 Yes 2002 160 -88 13\n#> 10 Yes 0.871 0.129 Yes 2020 39 9 6\n#> # ℹ 8,739 more rows\n#> # ℹ 14 more variables: tree_no_tree <fct>, dew_temp <dbl>, precip_annual <dbl>,\n#> # temp_annual_mean <dbl>, temp_annual_min <dbl>, temp_annual_max <dbl>,\n#> # temp_january_min <dbl>, vapor_min <dbl>, vapor_max <dbl>,\n#> # canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>, county <fct>" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-1", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-1", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" }, { - "objectID": "slides/intro-04-evaluating-models.html#confusion-matrix", - "href": "slides/intro-04-evaluating-models.html#confusion-matrix", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-2", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-2", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "slides/intro-04-evaluating-models.html#confusion-matrix-1", - "href": "slides/intro-04-evaluating-models.html#confusion-matrix-1", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) |>\n conf_mat(truth = forested, estimate = .pred_class)\n#> Truth\n#> Prediction Yes No\n#> Yes 5884 869\n#> No 438 1558" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-3", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-3", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() |>\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" }, { - "objectID": "slides/intro-04-evaluating-models.html#confusion-matrix-2", - "href": "slides/intro-04-evaluating-models.html#confusion-matrix-2", - "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) |>\n conf_mat(truth = forested, estimate = .pred_class) |>\n autoplot(type = \"heatmap\")" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-4", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-4", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() |>\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" }, { - "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance", - "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-5", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-5", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-1", - "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-1", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.931" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-6", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-6", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" }, { - "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-2", - "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-2", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.931\n\n\n\naugment(forested_fit, new_data = forested_train) |>\n specificity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.642" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-7", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-7", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree() |> \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-3", - "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-3", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\nforested_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(forested_fit, new_data = forested_train) |>\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n#> 2 specificity binary 0.642\n#> 3 sensitivity binary 0.931" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-8", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-8", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-4", - "href": "slides/intro-04-evaluating-models.html#metrics-for-model-performance-4", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \nMetrics and metric sets work with grouped data frames!\n\naugment(forested_fit, new_data = forested_train) |>\n group_by(tree_no_tree) |>\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 2 × 4\n#> tree_no_tree .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 Tree accuracy binary 0.875\n#> 2 No tree accuracy binary 0.807" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-1", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-1", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a logistic regression model.\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/\n\n\nExtension/Challenge: Edit this code to use a different model. For example, try using a conditional inference tree as implemented in the partykit package by changing the engine - or try an entirely different model type!\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-04-evaluating-models.html#your-turn", - "href": "slides/intro-04-evaluating-models.html#your-turn", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\n\nHow would you combine TP, TN, FP, and FN into one number to describe how often the model is correct?\n\n\n\n\n\n\n\n−+\n05:00" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#models-well-be-using-today", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#models-well-be-using-today", + "title": "3 - What makes a model?", + "section": "Models we’ll be using today", + "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" }, { - "objectID": "slides/intro-04-evaluating-models.html#two-class-data", - "href": "slides/intro-04-evaluating-models.html#two-class-data", - "title": "4 - Evaluating models", - "section": "Two class data", - "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "slides/intro-04-evaluating-models.html#varying-the-threshold", - "href": "slides/intro-04-evaluating-models.html#varying-the-threshold", - "title": "4 - Evaluating models", - "section": "Varying the threshold", - "text": "Varying the threshold" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression-1", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression-1", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "slides/intro-04-evaluating-models.html#roc-curves", - "href": "slides/intro-04-evaluating-models.html#roc-curves", - "title": "4 - Evaluating models", - "section": "ROC curves", - "text": "ROC curves\n\n\nFor an ROC (receiver operator characteristic) curve, we plot\n\nthe false positive rate (1 - specificity) on the x-axis\nthe true positive rate (sensitivity) on the y-axis\n\nwith sensitivity and specificity calculated at all possible thresholds.\n\n\n\n\n\n\n\n\n\n\n\n\nROC curves are insensitive to class imbalance." + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression-2", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression-2", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nFind a sigmoid line that separates the two classes" }, { - "objectID": "slides/intro-04-evaluating-models.html#roc-curves-1", - "href": "slides/intro-04-evaluating-models.html#roc-curves-1", - "title": "4 - Evaluating models", - "section": "ROC curves", - "text": "ROC curves\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "slides/intro-04-evaluating-models.html#roc-curves-2", - "href": "slides/intro-04-evaluating-models.html#roc-curves-2", - "title": "4 - Evaluating models", - "section": "ROC curves ", - "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(forested_fit, new_data = forested_train) |> \n roc_curve(truth = forested, .pred_Yes) |>\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.143 0.267 0.990\n#> 3 0.385 0.571 0.951\n\naugment(forested_fit, new_data = forested_train) |> \n roc_auc(truth = forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.881" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees-1", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees-1", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" }, { - "objectID": "slides/intro-04-evaluating-models.html#roc-curve-plot", - "href": "slides/intro-04-evaluating-models.html#roc-curve-plot", - "title": "4 - Evaluating models", - "section": "ROC curve plot ", - "text": "ROC curve plot \n\n\naugment(forested_fit, \n new_data = forested_train) |> \n roc_curve(truth = forested, \n .pred_Yes) |>\n autoplot()" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees-2", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees-2", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "slides/intro-04-evaluating-models.html#your-turn-1", - "href": "slides/intro-04-evaluating-models.html#your-turn-1", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nApply the forested_metrics metric set to augment()\noutput grouped by tree_no_tree.\nDo any metrics differ substantially between groups?\n\n\n\n−+\n05:00\n\n\n\n\nThe specificity for \"Tree\" is a good bit lower than it is for \"No tree\".\nSpecificity is the proportion of negatives that are correctly identified as negatives. “Negative” is the non-event level of the outcome, i.e. “non-forested.” So, when this index classifies the plot as having a tree, the model does not do well at correctly identifying the plot as non-forested when it is indeed non-forested." + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "title": "3 - What makes a model?", + "section": "All models are wrong, but some are useful!", + "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" }, { - "objectID": "slides/intro-04-evaluating-models.html#brier-score", - "href": "slides/intro-04-evaluating-models.html#brier-score", - "title": "4 - Evaluating models", - "section": "Brier score", - "text": "Brier score\nWhat if we don’t turn predicted probabilities into class predictions?\n\nThe Brier score is analogous to the mean squared error in regression models:\n\\[\nBrier_{class} = \\frac{1}{N}\\sum_{i=1}^N\\sum_{k=1}^C (y_{ik} - \\hat{p}_{ik})^2\n\\]" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "title": "3 - What makes a model?", + "section": "Workflows bind preprocessors and models", + "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" }, { - "objectID": "slides/intro-04-evaluating-models.html#brier-score-1", - "href": "slides/intro-04-evaluating-models.html#brier-score-1", - "title": "4 - Evaluating models", - "section": "Brier score", - "text": "Brier score\n\naugment(forested_fit, new_data = forested_train) |> \n brier_class(truth = forested, .pred_Yes) \n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.113\n\n\nSmaller values are better, for binary classification the “bad model threshold” is about 0.25." + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#what-is-wrong-with-this", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#what-is-wrong-with-this", + "title": "3 - What makes a model?", + "section": "What is wrong with this?", + "text": "What is wrong with this?" }, { - "objectID": "slides/intro-04-evaluating-models.html#separation-vs-calibration", - "href": "slides/intro-04-evaluating-models.html#separation-vs-calibration", - "title": "4 - Evaluating models", - "section": "Separation vs calibration", - "text": "Separation vs calibration\n\n\nThe ROC captures separation.\n\n\n\n\n\n\n\n\n\n\nThe Brier score captures calibration.\n\n\n\n\n\n\n\n\n\n\n\n\nGood separation: the densities don’t overlap.\nGood calibration: the calibration line follows the diagonal.\n\nCalibration plot: We bin observations according to predicted probability. In the bin for 20%-30% predicted prob, we should see an event rate of ~25% if the model is well-calibrated." + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#why-a-workflow", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#why-a-workflow", + "title": "3 - What makes a model?", + "section": "Why a workflow()? ", + "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering in Advanced tidymodels!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" }, { - "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-1", - "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-1", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting", - "text": "Dangers of overfitting" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-1", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-1", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\ntree_spec |> \n fit(forested ~ ., data = forested_train) \n#> parsnip model object\n#> \n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" }, { - "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-2", - "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-2", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-2", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-2", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\nworkflow() |>\n add_formula(forested ~ .) |>\n add_model(tree_spec) |>\n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" }, { - "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-3", - "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-3", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\nforested_fit |>\n augment(forested_train)\n#> # A tibble: 8,749 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness roughness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 0.931 0.0690 Yes 1997 66 82 10\n#> 2 Yes 0.983 0.0172 No 1997 284 -99 58\n#> 3 Yes 0.960 0.0401 Yes 2022 130 86 15\n#> 4 Yes 0.870 0.130 Yes 2021 202 -55 3\n#> 5 Yes 0.823 0.177 Yes 1995 75 -89 1\n#> 6 Yes 0.758 0.242 No 1995 110 -53 5\n#> 7 Yes 0.823 0.177 Yes 2022 111 73 12\n#> 8 No 0.467 0.533 Yes 1997 230 96 14\n#> 9 Yes 0.983 0.0172 Yes 2002 160 -88 13\n#> 10 Yes 0.871 0.129 Yes 2020 39 9 6\n#> # ℹ 8,739 more rows\n#> # ℹ 14 more variables: tree_no_tree <fct>, dew_temp <dbl>, precip_annual <dbl>,\n#> # temp_annual_mean <dbl>, temp_annual_min <dbl>, temp_annual_max <dbl>,\n#> # temp_january_min <dbl>, vapor_min <dbl>, vapor_max <dbl>,\n#> # canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>, county <fct>\n\nWe call this “resubstitution” or “repredicting the training set”" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-3", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-3", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\nworkflow(forested ~ ., tree_spec) |> \n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" }, { - "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-4", - "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-4", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n\nWe call this a “resubstitution estimate”" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-2", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-2", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code to make a workflow with your own model of choice.\n\nExtension/Challenge: Other than formulas, what kinds of preprocessors are supported?\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-5", - "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-5", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#predict-with-your-model", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#predict-with-your-model", + "title": "3 - What makes a model?", + "section": "Predict with your model ", + "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(forested ~ ., tree_spec) |> \n fit(data = forested_train)" }, { - "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-6", - "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-6", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n\n\n\nforested_fit |>\n augment(forested_test) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.711\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-3", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-3", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = forested_test)\nWhat do you notice about the structure of the result?\n\n\n\n−+\n03:00" }, { - "objectID": "slides/intro-04-evaluating-models.html#your-turn-2", - "href": "slides/intro-04-evaluating-models.html#your-turn-2", - "title": "4 - Evaluating models", + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-4", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-4", + "title": "3 - What makes a model?", "section": "Your turn", - "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" + "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = forested_test)\nHow does the output compare to the output from predict()?\n\n\n\n−+\n03:00" }, { - "objectID": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-7", - "href": "slides/intro-04-evaluating-models.html#dangers-of-overfitting-7", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n roc_auc(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.881\n\n\n\nforested_fit |>\n augment(forested_test) |>\n roc_auc(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.703\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?" }, { - "objectID": "slides/intro-04-evaluating-models.html#cross-validation", - "href": "slides/intro-04-evaluating-models.html#cross-validation", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model-1", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model-1", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit |>\n extract_fit_engine() |>\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\n⚠️ Never predict() with any extracted components!\n\nroundint = FALSE is only to quiet a warning" }, { - "objectID": "slides/intro-04-evaluating-models.html#cross-validation-1", - "href": "slides/intro-04-evaluating-models.html#cross-validation-1", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model-2", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model-2", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" }, { - "objectID": "slides/intro-04-evaluating-models.html#your-turn-3", - "href": "slides/intro-04-evaluating-models.html#your-turn-3", - "title": "4 - Evaluating models", + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-5", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-5", + "title": "3 - What makes a model?", "section": "Your turn", - "text": "Your turn\n\nUse augment() and a metric function to compute a classification metric like roc_auc().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" + "text": "Your turn\n\n\nExtract the model engine object from your fitted workflow and check it out.\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" }, { - "objectID": "slides/intro-04-evaluating-models.html#cross-validation-2", - "href": "slides/intro-04-evaluating-models.html#cross-validation-2", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(forested_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7874/875]> Fold01\n#> 2 <split [7874/875]> Fold02\n#> 3 <split [7874/875]> Fold03\n#> 4 <split [7874/875]> Fold04\n#> 5 <split [7874/875]> Fold05\n#> 6 <split [7874/875]> Fold06\n#> 7 <split [7874/875]> Fold07\n#> 8 <split [7874/875]> Fold08\n#> 9 <split [7874/875]> Fold09\n#> 10 <split [7875/874]> Fold10" + "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#the-whole-game---status-update", + "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#the-whole-game---status-update", + "title": "3 - What makes a model?", + "section": "The whole game - status update", + "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" }, { - "objectID": "slides/intro-04-evaluating-models.html#cross-validation-3", - "href": "slides/intro-04-evaluating-models.html#cross-validation-3", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWhat is in this?\n\nforested_folds <- vfold_cv(forested_train)\nforested_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n\n\nTalk about a list column, storing non-atomic types in dataframe" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#who-are-you", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have some exposure to basic statistical concepts like linear models and residuals\nYou do not need intermediate or expert familiarity with modeling or ML" }, { - "objectID": "slides/intro-04-evaluating-models.html#cross-validation-4", - "href": "slides/intro-04-evaluating-models.html#cross-validation-4", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(forested_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [6999/1750]> Fold1\n#> 2 <split [6999/1750]> Fold2\n#> 3 <split [6999/1750]> Fold3\n#> 4 <split [6999/1750]> Fold4\n#> 5 <split [7000/1749]> Fold5" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#who-are-tidymodels", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "slides/intro-04-evaluating-models.html#cross-validation-5", - "href": "slides/intro-04-evaluating-models.html#cross-validation-5", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nforested_folds <- vfold_cv(forested_train, v = 10)\nforested_folds\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7874/875]> Fold01\n#> 2 <split [7874/875]> Fold02\n#> 3 <split [7874/875]> Fold03\n#> 4 <split [7874/875]> Fold04\n#> 5 <split [7874/875]> Fold05\n#> 6 <split [7874/875]> Fold06\n#> 7 <split [7874/875]> Fold07\n#> 8 <split [7874/875]> Fold08\n#> 9 <split [7874/875]> Fold09\n#> 10 <split [7875/874]> Fold10\n\n\nSet the seed when creating resamples" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-2", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-2", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "slides/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", - "href": "slides/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", - "title": "4 - Evaluating models", - "section": "Fit our model to the resamples", - "text": "Fit our model to the resamples\n\nforested_res <- fit_resamples(forested_wflow, forested_folds)\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 4]>\n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 4]>" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-3", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-3", + "title": "1 - Introduction", + "section": "👀", + "text": "👀\n\n\nhttps://workshops.tidymodels.org" }, { - "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance", - "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nforested_res |>\n collect_metrics()\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.704 10 0.00653 pre0_mod0_post0\n#> 2 brier_class binary 0.214 10 0.00349 pre0_mod0_post0\n#> 3 roc_auc binary 0.692 10 0.00496 pre0_mod0_post0\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nWe can reliably measure performance using only the training data 🎉" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#plan-for-this-workshop", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Plan for this workshop", + "text": "Plan for this workshop\n\nYour data budget\nWhat makes a model\nEvaluating models\nTuning models" }, { - "objectID": "slides/intro-04-evaluating-models.html#comparing-metrics", - "href": "slides/intro-04-evaluating-models.html#comparing-metrics", - "title": "4 - Evaluating models", - "section": "Comparing metrics ", - "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\nforested_res |>\n collect_metrics() |> \n select(.metric, mean, n)\n#> # A tibble: 3 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.704 10\n#> 2 brier_class 0.214 10\n#> 3 roc_auc 0.692 10\n\n\nThe ROC AUC previously was\n\n0.88 for the training set\n0.7 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-4", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-4", + "title": "1 - Introduction", + "section": "", + "text": "Introduce yourself to your neighbors 👋" }, { - "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance-1", - "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance-1", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_forested <- control_resamples(save_pred = TRUE)\nforested_res <- fit_resamples(forested_wflow, forested_folds, control = ctrl_forested)\n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 4]> <tibble>" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" }, { - "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance-2", - "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance-2", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nforested_preds <- collect_predictions(forested_res)\nforested_preds\n#> # A tibble: 8,749 × 7\n#> .pred_class .pred_Yes .pred_No id forested .row .config \n#> <fct> <dbl> <dbl> <chr> <fct> <int> <chr> \n#> 1 Yes 0.953 0.0473 Fold01 No 2 pre0_mod0_post0\n#> 2 Yes 0.6 0.4 Fold01 No 6 pre0_mod0_post0\n#> 3 Yes 0.848 0.152 Fold01 Yes 7 pre0_mod0_post0\n#> 4 Yes 0.941 0.0588 Fold01 Yes 36 pre0_mod0_post0\n#> 5 Yes 0.895 0.105 Fold01 No 38 pre0_mod0_post0\n#> 6 Yes 1 0 Fold01 Yes 58 pre0_mod0_post0\n#> 7 No 0.187 0.812 Fold01 No 69 pre0_mod0_post0\n#> 8 Yes 0.905 0.0952 Fold01 Yes 71 pre0_mod0_post0\n#> 9 Yes 0.907 0.0930 Fold01 Yes 74 pre0_mod0_post0\n#> 10 Yes 0.904 0.0962 Fold01 Yes 80 pre0_mod0_post0\n#> # ℹ 8,739 more rows" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-1", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-1", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance-3", - "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance-3", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nforested_preds |> \n group_by(id) |>\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.729\n#> 2 Fold02 accuracy binary 0.685\n#> 3 Fold03 accuracy binary 0.687\n#> 4 Fold04 accuracy binary 0.711\n#> 5 Fold05 accuracy binary 0.736\n#> 6 Fold06 accuracy binary 0.670\n#> 7 Fold07 accuracy binary 0.699\n#> 8 Fold08 accuracy binary 0.710\n#> 9 Fold09 accuracy binary 0.699\n#> 10 Fold10 accuracy binary 0.719\n#> # ℹ 20 more rows" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-2024-edition", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-2024-edition", + "title": "1 - Introduction", + "section": "What is machine learning? (2024 edition)", + "text": "What is machine learning? (2024 edition)\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/\n\n\nIn the early 2010s, “Artificial intelligence” (AI) was largely synonymous with what we’ll refer to as “machine learning” in this workshop. In the late 2010s and early 2020s, AI usually referred to deep learning methods. Since the release of ChatGPT in late 2022, “AI” has come to also encompass large language models / generative models." }, { - "objectID": "slides/intro-04-evaluating-models.html#where-are-the-fitted-models", - "href": "slides/intro-04-evaluating-models.html#where-are-the-fitted-models", - "title": "4 - Evaluating models", - "section": "Where are the fitted models? ", - "text": "Where are the fitted models? \n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 4]> <tibble> \n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 4]> <tibble>\n\n\n🗑️" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-2", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-2", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "slides/intro-04-evaluating-models.html#bootstrapping", - "href": "slides/intro-04-evaluating-models.html#bootstrapping", - "title": "4 - Evaluating models", - "section": "Bootstrapping", - "text": "Bootstrapping" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#your-turn", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#your-turn", + "title": "1 - Introduction", + "section": "Your turn", + "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" }, { - "objectID": "slides/intro-04-evaluating-models.html#bootstrapping-1", - "href": "slides/intro-04-evaluating-models.html#bootstrapping-1", - "title": "4 - Evaluating models", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(forested_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [8749/3218]> Bootstrap01\n#> 2 <split [8749/3264]> Bootstrap02\n#> 3 <split [8749/3220]> Bootstrap03\n#> 4 <split [8749/3208]> Bootstrap04\n#> 5 <split [8749/3230]> Bootstrap05\n#> 6 <split [8749/3197]> Bootstrap06\n#> 7 <split [8749/3193]> Bootstrap07\n#> 8 <split [8749/3226]> Bootstrap08\n#> 9 <split [8749/3243]> Bootstrap09\n#> 10 <split [8749/3233]> Bootstrap10\n#> # ℹ 15 more rows" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-tidymodels", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-tidymodels", + "title": "1 - Introduction", + "section": "What is tidymodels? ", + "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.3.0 ──\n#> ✔ broom 1.0.9 ✔ rsample 1.3.1 \n#> ✔ dials 1.4.1 ✔ tibble 3.3.0 \n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1 \n#> ✔ infer 1.0.9 ✔ tune 1.3.0 \n#> ✔ modeldata 1.5.1 ✔ workflows 1.2.0.9002\n#> ✔ parsnip 1.3.2.9000 ✔ workflowsets 1.1.1 \n#> ✔ purrr 1.1.0 ✔ yardstick 1.3.2 \n#> ✔ recipes 1.3.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()" }, { - "objectID": "slides/intro-04-evaluating-models.html#the-whole-game---status-update", - "href": "slides/intro-04-evaluating-models.html#the-whole-game---status-update", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\nRoadmap for today\nMinimal version of predictive modeling process\nFeature engineering and tuning as iterative extensions" }, { - "objectID": "slides/intro-04-evaluating-models.html#your-turn-4", - "href": "slides/intro-04-evaluating-models.html#your-turn-4", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-1", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-1", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "slides/intro-04-evaluating-models.html#monte-carlo-cross-validation", - "href": "slides/intro-04-evaluating-models.html#monte-carlo-cross-validation", - "title": "4 - Evaluating models", - "section": "Monte Carlo Cross-Validation ", - "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(forested_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6561/2188]> Resample01\n#> 2 <split [6561/2188]> Resample02\n#> 3 <split [6561/2188]> Resample03\n#> 4 <split [6561/2188]> Resample04\n#> 5 <split [6561/2188]> Resample05\n#> 6 <split [6561/2188]> Resample06\n#> 7 <split [6561/2188]> Resample07\n#> 8 <split [6561/2188]> Resample08\n#> 9 <split [6561/2188]> Resample09\n#> 10 <split [6561/2188]> Resample10" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-2", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-2", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." }, { - "objectID": "slides/intro-04-evaluating-models.html#validation-set", - "href": "slides/intro-04-evaluating-models.html#validation-set", - "title": "4 - Evaluating models", - "section": "Validation set ", - "text": "Validation set \n\nset.seed(853)\nforested_val_split <- initial_validation_split(forested)\nvalidation_set(forested_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [4264/1421]> validation\n\n\nA validation set is just another type of resample" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-3", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-3", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "slides/intro-04-evaluating-models.html#random-forest-1", - "href": "slides/intro-04-evaluating-models.html#random-forest-1", - "title": "4 - Evaluating models", - "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this later!), as long as there are enough trees" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-4", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-4", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "slides/intro-04-evaluating-models.html#create-a-random-forest-model", - "href": "slides/intro-04-evaluating-models.html#create-a-random-forest-model", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-5", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-5", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "slides/intro-04-evaluating-models.html#create-a-random-forest-model-1", - "href": "slides/intro-04-evaluating-models.html#create-a-random-forest-model-1", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-6", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-6", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "slides/intro-04-evaluating-models.html#your-turn-5", - "href": "slides/intro-04-evaluating-models.html#your-turn-5", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the functions)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-7", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-7", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "slides/intro-04-evaluating-models.html#evaluating-model-performance-4", - "href": "slides/intro-04-evaluating-models.html#evaluating-model-performance-4", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nctrl_forested <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, forested_folds, control = ctrl_forested)\ncollect_metrics(rf_res)\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.755 10 0.00482 pre0_mod0_post0\n#> 2 brier_class binary 0.167 10 0.00321 pre0_mod0_post0\n#> 3 roc_auc binary 0.757 10 0.0103 pre0_mod0_post0" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#lets-install-some-packages", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"lightgbm\", \"lme4\", \"pak\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\npak::pak(\"simonpcouch/forested\")" }, { - "objectID": "slides/intro-04-evaluating-models.html#the-whole-game---status-update-1", - "href": "slides/intro-04-evaluating-models.html#the-whole-game---status-update-1", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#our-versions", + "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nR version 4.5.0 (2025-04-11), Quarto (1.6.42)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.4.0\n\n\nbroom\n1.0.9\n\n\nCubist\n0.5.0\n\n\ndials\n1.4.1\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.4\n\n\nembed\n1.1.5.9000\n\n\nfinetune\n1.2.1.9000\n\n\nforested\n0.2.0\n\n\nFormula\n1.2-5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nggplot2\n3.5.2\n\n\nlattice\n0.22-7\n\n\nlightgbm\n4.6.0\n\n\nlme4\n1.1-37\n\n\nmodeldata\n1.5.1\n\n\nparallelly\n1.45.1\n\n\nparsnip\n1.3.2.9000\n\n\nplotmo\n3.6.4\n\n\nplotrix\n3.8-4\n\n\nplumber\n1.3.0\n\n\nprobably\n1.1.0\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\npurrr\n1.1.0\n\n\nranger\n0.17.0\n\n\nrecipes\n1.3.1\n\n\nrpart\n4.1.24\n\n\nrpart.plot\n3.1.3\n\n\nrsample\n1.3.1\n\n\nrules\n1.0.2\n\n\nscales\n1.4.0\n\n\nsplines2\n0.5.4\n\n\nstacks\n1.1.1.9000\n\n\ntext2vec\n0.6.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntextrecipes\n1.1.0.9000\n\n\ntibble\n3.3.0\n\n\ntidymodels\n1.3.0\n\n\ntidyr\n1.3.1\n\n\ntune\n1.3.0\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.2.0.9002\n\n\nworkflowsets\n1.1.1\n\n\nyardstick\n1.3.2" }, { - "objectID": "slides/intro-04-evaluating-models.html#the-final-fit", - "href": "slides/intro-04-evaluating-models.html#the-final-fit", - "title": "4 - Evaluating models", - "section": "The final fit ", - "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# forested_split has train + test info\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [8749/2188]> train/test split <tibble> <tibble> <tibble> <workflow>" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#chicago-l-train-data", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#chicago-l-train-data", + "title": "Case Study on Transportation", + "section": "Chicago L-Train data", + "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" }, { - "objectID": "slides/intro-04-evaluating-models.html#what-is-in-final_fit", - "href": "slides/intro-04-evaluating-models.html#what-is-in-final_fit", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.761 pre0_mod0_post0\n#> 2 roc_auc binary 0.761 pre0_mod0_post0\n#> 3 brier_class binary 0.162 pre0_mod0_post0\n\n\nThese are metrics computed with the test set" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#predictors", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#predictors", + "title": "Case Study on Transportation", + "section": "Predictors", + "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." + }, + { + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#l-train-locations", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#l-train-locations", + "title": "Case Study on Transportation", + "section": "L Train Locations", + "text": "L Train Locations" }, { - "objectID": "slides/intro-04-evaluating-models.html#what-is-in-final_fit-1", - "href": "slides/intro-04-evaluating-models.html#what-is-in-final_fit-1", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 2,188 × 7\n#> .pred_class .pred_Yes .pred_No id forested .row .config \n#> <fct> <dbl> <dbl> <chr> <fct> <int> <chr> \n#> 1 Yes 0.957 0.0429 train/test split Yes 4 pre0_mod0_pos…\n#> 2 Yes 0.810 0.190 train/test split Yes 8 pre0_mod0_pos…\n#> 3 Yes 0.826 0.174 train/test split Yes 10 pre0_mod0_pos…\n#> 4 Yes 0.842 0.158 train/test split Yes 19 pre0_mod0_pos…\n#> 5 Yes 0.863 0.137 train/test split Yes 23 pre0_mod0_pos…\n#> 6 Yes 0.585 0.415 train/test split No 28 pre0_mod0_pos…\n#> 7 Yes 0.847 0.153 train/test split Yes 34 pre0_mod0_pos…\n#> 8 Yes 0.605 0.395 train/test split No 35 pre0_mod0_pos…\n#> 9 Yes 0.964 0.0362 train/test split Yes 38 pre0_mod0_pos…\n#> 10 Yes 0.926 0.0744 train/test split Yes 40 pre0_mod0_pos…\n#> # ℹ 2,178 more rows" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#your-turn-explore-the-data", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#your-turn-explore-the-data", + "title": "Case Study on Transportation", + "section": "Your turn: Explore the Data", + "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n−+\n05:00" }, { - "objectID": "slides/intro-04-evaluating-models.html#what-is-in-final_fit-2", - "href": "slides/intro-04-evaluating-models.html#what-is-in-final_fit-2", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 8749 \n#> Number of independent variables: 18 \n#> Mtry: 4 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.1671606\n\n\nUse this for prediction on new data, like for deploying" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#splitting-with-chicago-data", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#splitting-with-chicago-data", + "title": "Case Study on Transportation", + "section": "Splitting with Chicago data ", + "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" }, { - "objectID": "slides/intro-04-evaluating-models.html#the-whole-game", - "href": "slides/intro-04-evaluating-models.html#the-whole-game", - "title": "4 - Evaluating models", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#time-series-resampling", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#time-series-resampling", + "title": "Case Study on Transportation", + "section": "Time series resampling", + "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" }, { - "objectID": "slides/intro-06-wrapping-up.html#your-turn", - "href": "slides/intro-06-wrapping-up.html#your-turn", - "title": "6 - Wrapping up", - "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#rolling-forecast-origin-resampling", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#rolling-forecast-origin-resampling", + "title": "Case Study on Transportation", + "section": "Rolling forecast origin resampling", + "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." }, { - "objectID": "slides/intro-06-wrapping-up.html#resources-to-keep-learning", - "href": "slides/intro-06-wrapping-up.html#resources-to-keep-learning", - "title": "6 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://tidymodels.aml4td.org/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Bluesky, Mastodon and at the tidyverse blog for updates!" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." }, { - "objectID": "slides/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", - "href": "slides/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", - "title": "Extras - workflowsets", - "section": "How can we compare multiple model workflows at once?", - "text": "How can we compare multiple model workflows at once?" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-1", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-1", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." }, { - "objectID": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set", - "href": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set", - "title": "Extras - workflowsets", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nwf_set <- workflow_set(list(forested ~ .), list(tree_spec, rf_spec))\nwf_set\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-2", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-2", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" }, { - "objectID": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", - "href": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", - "title": "Extras - workflowsets", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nwf_set_fit <- wf_set |>\n workflow_map(\"fit_resamples\", resamples = forested_folds)\nwf_set_fit\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-3", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-3", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" }, { - "objectID": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", - "href": "slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", - "title": "Extras - workflowsets", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nwf_set_fit |>\n rank_results()\n#> # A tibble: 6 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_for… pre0_m… accura… 0.753 0.00462 10 formula rand… 1\n#> 2 formula_rand_for… pre0_m… brier_… 0.167 0.00312 10 formula rand… 1\n#> 3 formula_rand_for… pre0_m… roc_auc 0.757 0.00994 10 formula rand… 1\n#> 4 formula_decision… pre0_m… accura… 0.704 0.00653 10 formula deci… 2\n#> 5 formula_decision… pre0_m… brier_… 0.214 0.00349 10 formula deci… 2\n#> 6 formula_decision… pre0_m… roc_auc 0.692 0.00496 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-4", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-4", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] |> assessment() |> pluck(\"date\") |> range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] |> assessment() |> pluck(\"date\") |> range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" }, { - "objectID": "slides/intro-extra-workflowsets.html#your-turn", - "href": "slides/intro-extra-workflowsets.html#your-turn", - "title": "Extras - workflowsets", - "section": "Your turn", - "text": "Your turn\n\nWhen do you think a workflow set would be useful?\nDiscuss with your neighbors!\n\n\n\n−+\n03:00" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#our-resampling-object", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#our-resampling-object", + "title": "Case Study on Transportation", + "section": "Our resampling object ", + "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching", - "title": "2 - Your data budget", - "section": "Data on tree frog hatching", - "text": "Data on tree frog hatching" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#feature-engineering-with-recipes", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#feature-engineering-with-recipes", + "title": "Case Study on Transportation", + "section": "Feature engineering with recipes ", + "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-1", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-1", - "title": "2 - Your data budget", - "section": "Data on tree frog hatching", - "text": "Data on tree frog hatching\n\nRed-eyed tree frog embryos can hatch earlier than their normal ~7 days if they detect potential predator threat!\nType ?stacks::tree_frogs to learn more about this dataset, including references.\nWe are using a slightly modified version from stacks.\n\n\nlibrary(tidymodels)\n\ndata(\"tree_frogs\", package = \"stacks\")\ntree_frogs <- tree_frogs %>%\n mutate(t_o_d = factor(t_o_d),\n age = age / 86400) %>%\n filter(!is.na(latency)) %>%\n select(-c(clutch, hatched))" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe", + "title": "Case Study on Transportation", + "section": "A recipe", + "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-2", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-2", - "title": "2 - Your data budget", - "section": "Data on tree frog hatching", - "text": "Data on tree frog hatching\n\n\n\nN = 572\nA numeric outcome, latency\n4 other variables\n\ntreatment, reflex, and t_o_d are nominal predictors\nage is a numeric predictor\n\n\n\n\n\n\n\nlatency: How long it took the frog to hatch after being stimulated - i.e. after being poked by a blunt probe (in seconds).\ntreatment: Whether or not they got gentamicin, a compound that knocks out the embryo’s lateral line (a sensory organ).\nreflex: A measure of ear function (low, mid, full)\nt_o_d: Time that the stimulus was applied (morning, afternoon, night)\nage: Age at the time it was stimulated (in days)" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) |> \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-3", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-on-tree-frog-hatching-3", - "title": "2 - Your data budget", - "section": "Data on tree frog hatching", - "text": "Data on tree frog hatching\n\ntree_frogs\n#> # A tibble: 572 × 5\n#> treatment reflex age t_o_d latency\n#> <chr> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.40 morning 22\n#> 2 control low 4.18 night 360\n#> 3 control full 4.65 afternoon 106\n#> 4 control mid 4.14 night 180\n#> 5 control full 4.6 afternoon 60\n#> 6 gentamicin full 5.36 morning 39\n#> 7 control full 4.56 afternoon 214\n#> 8 control full 5.43 morning 50\n#> 9 control full 4.63 afternoon 224\n#> 10 control full 5.40 morning 63\n#> # … with 562 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates-1", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates-1", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) |> \n step_date(date, features = c(\"dow\", \"month\", \"year\")) |> \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates-2", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates-2", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) |> \n step_date(date, features = c(\"dow\", \"month\", \"year\")) |> \n step_holiday(date) |> \n update_role(date, new_role = \"id\") |>\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points. The help page has a good discussion about the nuances." }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-1", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-1", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---remove-constant-columns", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---remove-constant-columns", + "title": "Case Study on Transportation", + "section": "A recipe - remove constant columns ", + "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) |> \n step_date(date, features = c(\"dow\", \"month\", \"year\")) |> \n step_holiday(date) |> \n update_role(date, new_role = \"id\") |>\n update_role_requirements(role = \"id\", bake = TRUE) |> \n step_zv(all_nominal_predictors())" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-2", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-2", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---handle-correlations", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---handle-correlations", + "title": "Case Study on Transportation", + "section": "A recipe - handle correlations ", + "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec |> \n step_normalize(all_of(!!stations)) |> \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n03:00" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#make-some-models", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#make-some-models", + "title": "Case Study on Transportation", + "section": "Make some models ", + "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) |> set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) |> \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-3", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-3", - "title": "2 - Your data budget", - "section": "Data splitting and spending ", - "text": "Data splitting and spending \n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs)\nfrog_split\n#> <Training/Testing/Total>\n#> <429/143/572>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#process-them-on-the-resamples", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#process-them-on-the-resamples", + "title": "Case Study on Transportation", + "section": "Process them on the resamples", + "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set |> \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#accessing-the-data", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#accessing-the-data", - "title": "2 - Your data budget", - "section": "Accessing the data ", - "text": "Accessing the data \n\nfrog_train <- training(frog_split)\nfrog_test <- testing(frog_split)" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#how-do-the-results-look", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#how-do-the-results-look", + "title": "Case Study on Transportation", + "section": "How do the results look?", + "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 34 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.804 0.101 16 recipe cubis… 1\n#> 2 basic_cubist Preprocessor1_Model1 mae 0.901 0.115 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor2_Model2 mae 0.985 0.120 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model2 mae 0.990 0.121 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor1_Model3 mae 1.00 0.119 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor4_Model1 mae 1.02 0.131 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor3_Model3 mae 1.03 0.121 16 recipe cubis… 7\n#> 8 pca_cubist Preprocessor3_Model2 mae 1.07 0.137 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model9 mae 1.07 0.115 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model8 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 24 more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#the-training-set", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#the-training-set", - "title": "2 - Your data budget", - "section": "The training set", - "text": "The training set\n\nfrog_train\n#> # A tibble: 429 × 5\n#> treatment reflex age t_o_d latency\n#> <chr> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.36 morning 36\n#> 2 gentamicin full 5.37 morning 72\n#> 3 gentamicin full 4.65 afternoon 141\n#> 4 control full 5.42 morning 27\n#> 5 control full 5.43 morning 27\n#> 6 gentamicin full 5.38 morning 73\n#> 7 gentamicin full 5.42 morning 68\n#> 8 gentamicin full 4.75 afternoon 124\n#> 9 control full 5.00 night 62\n#> 10 control full 5.39 morning 25\n#> # … with 419 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#plot-the-results", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#plot-the-results", + "title": "Case Study on Transportation", + "section": "Plot the results ", + "text": "Plot the results \n\nautoplot(chi_res)" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#the-test-set", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#the-test-set", - "title": "2 - Your data budget", - "section": "The test set ", - "text": "The test set \n\nfrog_test\n#> # A tibble: 143 × 5\n#> treatment reflex age t_o_d latency\n#> <chr> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.40 morning 22\n#> 2 control low 4.18 night 360\n#> 3 control full 4.63 afternoon 224\n#> 4 gentamicin full 4.75 afternoon 158\n#> 5 control mid 4.22 night 91\n#> 6 gentamicin full 4.89 night 301\n#> 7 control full 5.38 morning 2\n#> 8 control full 4.80 afternoon 56\n#> 9 control full 5.36 morning 11\n#> 10 control full 5.40 morning 64\n#> # … with 133 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#pull-out-specific-results", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#pull-out-specific-results", + "title": "Case Study on Transportation", + "section": "Pull out specific results ", + "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res |> \n extract_workflow_set_result(\"pca_cubist\") |> \n autoplot()" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn-1", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn-1", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n05:00" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-4", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#data-splitting-and-spending-4", - "title": "2 - Your data budget", - "section": "Data splitting and spending ", - "text": "Data splitting and spending \n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs, prop = 0.8)\nfrog_train <- training(frog_split)\nfrog_test <- testing(frog_split)\n\nnrow(frog_train)\n#> [1] 457\nnrow(frog_test)\n#> [1] 115" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-1", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-1", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-1", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-1", - "title": "2 - Your data budget", - "section": "", - "text": "We will use this tomorrow" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-2", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-2", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" + }, + { + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-3", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-3", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn-2", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#your-turn-2", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nExplore the frog_train data on your own!\n\nWhat’s the distribution of the outcome, latency?\nWhat’s the distribution of numeric variables like age?\nHow does latency differ across the categorical variables?\n\n\n\n\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-4", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-4", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-3", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-3", - "title": "2 - Your data budget", - "section": "", - "text": "ggplot(frog_train, aes(latency)) +\n geom_histogram(bins = 20)\n\n\n\nThis histogram brings up a concern. What if in our training set we get unlucky and sample few or none of these large values? That could mean that our model wouldn’t be able to predict such values. Let’s come back to that!" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-5", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-5", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-4", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-4", - "title": "2 - Your data budget", - "section": "", - "text": "ggplot(frog_train, aes(latency, treatment, fill = treatment)) +\n geom_boxplot(alpha = 0.5, show.legend = FALSE)" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#building-a-model-stack", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#building-a-model-stack", + "title": "Case Study on Transportation", + "section": "Building a model stack ", + "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-5", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-5", - "title": "2 - Your data budget", - "section": "", - "text": "frog_train %>%\n ggplot(aes(latency, reflex, fill = reflex)) +\n geom_boxplot(alpha = 0.3, show.legend = FALSE)" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#start-the-stack-and-add-members", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#start-the-stack-and-add-members", + "title": "Case Study on Transportation", + "section": "Start the stack and add members ", + "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() |> \n add_candidates(chi_res)" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-6", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-6", - "title": "2 - Your data budget", - "section": "", - "text": "ggplot(frog_train, aes(age, latency, color = reflex)) +\n geom_point(alpha = .8, size = 2)" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#estimate-weights-for-each-candidate", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#estimate-weights-for-each-candidate", + "title": "Case Study on Transportation", + "section": "Estimate weights for each candidate ", + "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack, penalty = 10^seq(-6, -1, length.out = 25))\n\nchi_stack_res\n#> # A tibble: 5 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.354 \n#> 2 basic_cubist_1_1 cubist_rules 0.263 \n#> 3 pca_cubist_1_3 cubist_rules 0.203 \n#> 4 basic_cubist_1_6 cubist_rules 0.155 \n#> 5 pca_lm_2_1 linear_reg 0.0479" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#section-7", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#section-7", - "title": "2 - Your data budget", - "section": "", - "text": "Stratified sampling would split within each quartile\n\nBased on our exploration, we realized that stratifying by latency might help get a consistent distribution. For instance, we’d include high and low latency in both the test and training" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#how-did-it-do", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#how-did-it-do", + "title": "Case Study on Transportation", + "section": "How did it do? ", + "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" }, { - "objectID": "archive/2022-07-RStudio-conf/02-data-budget.html#stratification", - "href": "archive/2022-07-RStudio-conf/02-data-budget.html#stratification", - "title": "2 - Your data budget", - "section": "Stratification", - "text": "Stratification\nUse strata = latency\n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs, prop = 0.8, strata = latency)\nfrog_split\n#> <Training/Testing/Total>\n#> <456/116/572>\n\n\nStratification often helps, with very little downside" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#what-does-it-use", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#what-does-it-use", + "title": "Case Study on Transportation", + "section": "What does it use? ", + "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 rsq standard 0.380\n#> 3 mae standard 40.2\n\n\n\nRMSE: difference between the predicted and observed values ⬇️\n\\(R^2\\): squared correlation between the predicted and observed values ⬆️\nMAE: similar to RMSE, but mean absolute error ⬇️" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#fit-the-required-candidate-models", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#fit-the-required-candidate-models", + "title": "Case Study on Transportation", + "section": "Fit the required candidate models", + "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-1", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#the-test-set-best-cubist-model", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#the-test-set-best-cubist-model", + "title": "Case Study on Transportation", + "section": "The test set: best Cubist model ", + "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res |> \n extract_workflow_set_result(\"pca_cubist\") |> \n select_best()\n\ncubist_res <- \n chi_res |> \n extract_workflow(\"pca_cubist\") |> \n finalize_workflow(best_cubist) |> \n last_fit(split = chi_split, metrics = metric_set(mae))" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-2", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n group_by(reflex) %>%\n rmse(latency, .pred)\n#> # A tibble: 3 × 4\n#> reflex .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 low rmse standard 94.3\n#> 2 mid rmse standard 101. \n#> 3 full rmse standard 51.2" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#the-test-set-stack-ensemble", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#the-test-set-stack-ensemble", + "title": "Case Study on Transportation", + "section": "The test set: stack ensemble", + "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) |> \n bind_cols(chi_test)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-3", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#metrics-for-model-performance-3", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\nfrog_metrics <- metric_set(rmse, msd)\naugment(tree_fit, new_data = frog_test) %>%\n frog_metrics(latency, .pred)\n#> # A tibble: 2 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 msd standard -0.908" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#compare-the-results", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#compare-the-results", + "title": "Case Study on Transportation", + "section": "Compare the results ", + "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.669 Preprocessor1_Model1\n\nstack_pred |> mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.669" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-1", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#plot-the-test-set", + "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#plot-the-test-set", + "title": "Case Study on Transportation", + "section": "Plot the test set ", + "text": "Plot the test set \n\n\nlibrary(probably)\ncubist_res |> \n collect_predictions() |> \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-2", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2025-08-nyr/slides/annotations.html#section", + "href": "archive/2025-08-nyr/slides/annotations.html#section", + "title": "Annotations", + "section": "👀", + "text": "👀\nThis page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-3", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-3", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\ntree_fit %>%\n augment(frog_train)\n#> # A tibble: 456 × 7\n#> .pred .resid treatment reflex age t_o_d latency\n#> <dbl> <dbl> <fct> <fct> <dbl> <fct> <dbl>\n#> 1 39.8 -6.85 control full 5.42 morning 33\n#> 2 66.7 -47.7 control full 5.38 morning 19\n#> 3 66.7 -64.7 control full 5.38 morning 2\n#> 4 39.8 -0.848 control full 5.44 morning 39\n#> 5 39.8 2.15 control full 5.41 morning 42\n#> 6 59.8 -39.8 control full 4.75 afternoon 20\n#> 7 83.1 -52.1 control full 4.95 night 31\n#> 8 39.8 -18.8 control full 5.42 morning 21\n#> 9 64.6 -34.6 gentamicin full 5.39 morning 30\n#> 10 174. -131. control full 4.55 afternoon 43\n#> # ℹ 446 more rows\n\nWe call this “resubstitution” or “repredicting the training set”" + "objectID": "archive/2025-08-nyr/slides/annotations.html#data-splitting-and-spending", + "href": "archive/2025-08-nyr/slides/annotations.html#data-splitting-and-spending", + "title": "Annotations", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nMore about the initial data split can be found in Chapter 3 of Applied Machine Learning for Tabular Data (AML4TD).\nIn particular, a three-way split into training, validation, and testing set can be done via\n\nset.seed(123)\ninitial_validation_split(forested, prop = c(0.6, 0.2))\n#> <Training/Validation/Testing/Total>\n#> <4264/1421/1422/7107>" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-4", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-4", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4\n\nWe call this a “resubstitution estimate”" + "objectID": "archive/2025-08-nyr/slides/annotations.html#what-is-set.seed", + "href": "archive/2025-08-nyr/slides/annotations.html#what-is-set.seed", + "title": "Annotations", + "section": "What is set.seed()?", + "text": "What is set.seed()?\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-5", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-5", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4" + "objectID": "archive/2025-08-nyr/slides/annotations.html#what-is-wrong-with-this", + "href": "archive/2025-08-nyr/slides/annotations.html#what-is-wrong-with-this", + "title": "Annotations", + "section": "What is wrong with this?", + "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-6", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-6", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4\n\n\n\ntree_fit %>%\n augment(frog_test) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" + "objectID": "archive/2025-08-nyr/slides/annotations.html#brier-score", + "href": "archive/2025-08-nyr/slides/annotations.html#brier-score", + "title": "Annotations", + "section": "Brier score", + "text": "Brier score\nThe Brier score measures how close a model probability estimate is to its best possible value (i.e., zero or one).\nIn the best case, the model is perfect, and every prediction equals 0.0 or 1.0 (depending on the true class). In this case, the Brier score is zero.\nWhen the model is uninformative and there are two classes, the worst-case values range from 0.25 to about 0.50. Imagine that the model predicts the same noninformative prediction of 50% (basically “¯\\(ツ)/¯”). In that case, every prediction is either \\((0.00 - 0.50)^2\\) or \\((1.00 - 0.50)^2\\). The average of those is 0.25.\nThere are many different ways a model can be bad though, and some of these will produce Brier scores between 0.25 and 0.50." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse augment() and metrics() to compute a regression metric like mae().\nCompute the metrics for both training and testing data.\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" + "objectID": "archive/2025-08-nyr/slides/annotations.html#dangers-of-overfitting", + "href": "archive/2025-08-nyr/slides/annotations.html#dangers-of-overfitting", + "title": "Annotations", + "section": "Dangers of overfitting", + "text": "Dangers of overfitting\nSee the “Overfitting” chapter of AML4TD for more information." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-7", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#dangers-of-overfitting-7", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4 \n#> 2 rsq standard 0.494\n#> 3 mae standard 33.4\n\n\n\ntree_fit %>%\n augment(frog_test) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 rsq standard 0.380\n#> 3 mae standard 40.2\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" + "objectID": "archive/2025-08-nyr/slides/annotations.html#where-are-the-fitted-models", + "href": "archive/2025-08-nyr/slides/annotations.html#where-are-the-fitted-models", + "title": "Annotations", + "section": "Where are the fitted models?", + "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2025-08-nyr/slides/annotations.html#resampling-strategy", + "href": "archive/2025-08-nyr/slides/annotations.html#resampling-strategy", + "title": "Annotations", + "section": "Resampling Strategy", + "text": "Resampling Strategy\nThese data have a time component, and while not a typical time series data set, we have the option to use a time series resampling method.\nAn example is shown in the extra slides “Case Study on Transportation”.\nConsider the agent data. Cross-validation may not group all of an agent’s data into the analysis or assessment sets. In this case, our analysis data might have future data that is later than the agent’s data in the assessment set." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-1", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2025-08-nyr/slides/annotations.html#different-types-of-grids", + "href": "archive/2025-08-nyr/slides/annotations.html#different-types-of-grids", + "title": "Annotations", + "section": "Different types of grids", + "text": "Different types of grids\nMore on space-filling designs in Chapters 4 and 5 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences.\nIn the next version of tune (version 1.3.0) an improved set of space-filling designs will be the first choice if you ask for an automated grid." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-1", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" + "objectID": "archive/2025-08-nyr/slides/annotations.html#update-parameter-ranges", + "href": "archive/2025-08-nyr/slides/annotations.html#update-parameter-ranges", + "title": "Annotations", + "section": "Update parameter ranges", + "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object |> \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) |> \n step_spline_natural(dis, deg_free = tune()) |> \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-2", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(frog_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [410/46]> Fold01\n#> 2 <split [410/46]> Fold02\n#> 3 <split [410/46]> Fold03\n#> 4 <split [410/46]> Fold04\n#> 5 <split [410/46]> Fold05\n#> 6 <split [410/46]> Fold06\n#> 7 <split [411/45]> Fold07\n#> 8 <split [411/45]> Fold08\n#> 9 <split [411/45]> Fold09\n#> 10 <split [411/45]> Fold10" + "objectID": "archive/2025-08-nyr/slides/annotations.html#early-stopping-for-boosted-trees", + "href": "archive/2025-08-nyr/slides/annotations.html#early-stopping-for-boosted-trees", + "title": "Annotations", + "section": "Early stopping for boosted trees", + "text": "Early stopping for boosted trees\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the lightgbm package.\nbonsai has its own wrapper around (lightgbm::lgb.train()) called bonsai::train_lightgbm(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the lightgbm entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for lightgbm)." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-3", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-3", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWhat is in this?\n\nfrog_folds <- vfold_cv(frog_train)\nfrog_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n\n\nTalk about a list column, storing non-atomic types in dataframe" + "objectID": "archive/2025-08-nyr/slides/annotations.html#gaussian-processes-and-optimization", + "href": "archive/2025-08-nyr/slides/annotations.html#gaussian-processes-and-optimization", + "title": "Annotations", + "section": "Gaussian Processes and Optimization", + "text": "Gaussian Processes and Optimization\nSome other references for GP’s:\n\nChapter 5 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences\nBayesian Optimization, Chapter 3 (pdf)\nGaussian Processes for Machine Learning (pdf)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-4", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-4", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(frog_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [364/92]> Fold1\n#> 2 <split [365/91]> Fold2\n#> 3 <split [365/91]> Fold3\n#> 4 <split [365/91]> Fold4\n#> 5 <split [365/91]> Fold5" + "objectID": "archive/2025-08-nyr/slides/annotations.html#acquisition-functions", + "href": "archive/2025-08-nyr/slides/annotations.html#acquisition-functions", + "title": "Annotations", + "section": "Acquisition Functions", + "text": "Acquisition Functions\nMore references:\n\nChapter 7 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences\nBayesian Optimization, Chapter 6 (pdf)\nGaussian Processes for Machine Learning" }, - { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-5", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-5", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(frog_train, strata = latency)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [408/48]> Fold01\n#> 2 <split [408/48]> Fold02\n#> 3 <split [408/48]> Fold03\n#> 4 <split [409/47]> Fold04\n#> 5 <split [411/45]> Fold05\n#> 6 <split [412/44]> Fold06\n#> 7 <split [412/44]> Fold07\n#> 8 <split [412/44]> Fold08\n#> 9 <split [412/44]> Fold09\n#> 10 <split [412/44]> Fold10\n\n\nStratification often helps, with very little downside" + { + "objectID": "archive/2025-08-nyr/slides/annotations.html#per-agent-statistics", + "href": "archive/2025-08-nyr/slides/annotations.html#per-agent-statistics", + "title": "Annotations", + "section": "Per-agent statistics", + "text": "Per-agent statistics\nThe effect encoding method essentially takes the effect of a variable, like agent, and makes a data column for that effect. In our example, affect of the agent on the ADR is quantified by a model and then added as a data column to be used in the model.\nSuppose agent Max has a single reservation in the data and it had an ADR of €200. If we used a naive estimate for Max’s effect, the model is being told that Max should always produce an effect of €200. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned agent Davis, who has taken 250 reservations with an average ADR of €100. Davis’s mean is more predictive because it is estimated with better data (i.e., more total reservations). Partial pooling leverages the entire data set and can borrow strength from all of the agents. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a agent’s data is of good quality, the partial pooling effect estimate is closer to the raw mean. Max’s data is not great and is “shrunk” towards the center of the overall average. Since there is so little known about Max’s reservation history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nSince this example has a numeric outcome, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-6", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#cross-validation-6", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nfrog_folds <- vfold_cv(frog_train, v = 10, strata = latency)\nfrog_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [408/48]> Fold01\n#> 2 <split [408/48]> Fold02\n#> 3 <split [408/48]> Fold03\n#> 4 <split [409/47]> Fold04\n#> 5 <split [411/45]> Fold05\n#> 6 <split [412/44]> Fold06\n#> 7 <split [412/44]> Fold07\n#> 8 <split [412/44]> Fold08\n#> 9 <split [412/44]> Fold09\n#> 10 <split [412/44]> Fold10\n\n\nSet the seed when creating resamples" + "objectID": "archive/2025-08-nyr/slides/annotations.html#agent-effects", + "href": "archive/2025-08-nyr/slides/annotations.html#agent-effects", + "title": "Annotations", + "section": "Agent effects", + "text": "Agent effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage though." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#fit-our-model-to-the-resamples", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#fit-our-model-to-the-resamples", - "title": "4 - Evaluating models", - "section": "Fit our model to the resamples", - "text": "Fit our model to the resamples\n\ntree_res <- fit_resamples(tree_wflow, frog_folds)\ntree_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 4]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 4]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 4]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 4]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 4]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 4]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 4]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 4]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 4]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 4]>" + "objectID": "archive/2025-08-nyr/slides/annotations.html#a-recipe---handle-correlations", + "href": "archive/2025-08-nyr/slides/annotations.html#a-recipe---handle-correlations", + "title": "Annotations", + "section": "A recipe - handle correlations", + "text": "A recipe - handle correlations\nIn this code chunk, What’s the story with !!stations?\n\nchi_pca_rec <- \n chi_rec |> \n step_normalize(all_of(!!stations)) |> \n step_pca(all_of(!!stations), num_comp = tune())\n\nstations is a vector of names of 20 columns that we want to use in the steps. If the list were shorter, we could type them in (e.g., c(\"col1\", \"col2\") etc.).\nThe vector lives in our global workspace, and if we are in parallel, the worker processes might not have access to stations. The !! (frequently said as “bang bang”) inserts the actual contents of the vector into the all_of() calls so that it looks like you just typed it in.\nThis means that the parallel process workers have a copy of the data in their reach and the code will run without error." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\ntree_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 rmse standard 59.6 10 2.31 pre0_mod0_post0\n#> 2 rsq standard 0.305 10 0.0342 pre0_mod0_post0\n\n\nWe can reliably measure performance using only the training data 🎉" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#previously---setup", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#previously---setup", + "title": "5 - Iterative Search", + "section": "Previously - Setup", + "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#comparing-metrics", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#comparing-metrics", - "title": "4 - Evaluating models", - "section": "Comparing metrics ", - "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\ntree_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 rmse 59.6 10\n#> 2 rsq 0.305 10\n\n\nThe RMSE previously was\n\n49.36 for the training set\n59.16 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#previously---data-usage", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#previously---data-usage", + "title": "5 - Iterative Search", + "section": "Previously - Data Usage", + "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance-1", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_frog <- control_resamples(save_pred = TRUE)\ntree_res <- fit_resamples(tree_wflow, frog_folds, control = ctrl_frog)\n\ntree_preds <- collect_predictions(tree_res)\ntree_preds\n#> # A tibble: 456 × 5\n#> .pred id latency .row .config \n#> <dbl> <chr> <dbl> <int> <chr> \n#> 1 39.6 Fold01 33 1 pre0_mod0_post0\n#> 2 72.1 Fold01 2 3 pre0_mod0_post0\n#> 3 63.8 Fold01 30 9 pre0_mod0_post0\n#> 4 72.1 Fold01 46 13 pre0_mod0_post0\n#> 5 43.3 Fold01 11 28 pre0_mod0_post0\n#> 6 61.7 Fold01 41 35 pre0_mod0_post0\n#> 7 39.6 Fold01 43 51 pre0_mod0_post0\n#> 8 134. Fold01 20 70 pre0_mod0_post0\n#> 9 70.6 Fold01 21 74 pre0_mod0_post0\n#> 10 39.6 Fold01 14 106 pre0_mod0_post0\n#> # ℹ 446 more rows" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#our-boosting-model", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#our-boosting-model", + "title": "5 - Iterative Search", + "section": "Our Boosting Model", + "text": "Our Boosting Model\nWe used feature hashing to generate a smaller set of indicator columns to deal with the large number of levels for the agent and country predictors.\n\nTree-based models (and a few others) don’t require indicators for categorical predictors. They can split on these variables as-is.\n\nWe’ll keep all categorical predictors as factors and focus on optimizing additional boosting parameters." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-3", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-3", - "title": "4 - Evaluating models", - "section": "", - "text": "tree_preds %>% \n ggplot(aes(latency, .pred, color = id)) + \n geom_abline(lty = 2, col = \"gray\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#our-boosting-model-1", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#our-boosting-model-1", + "title": "5 - Iterative Search", + "section": "Our Boosting Model", + "text": "Our Boosting Model\n\nlgbm_spec <- \n boost_tree(trees = 1000, learn_rate = tune(), min_n = tune(), \n tree_depth = tune(), loss_reduction = tune(), \n stop_iter = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(avg_price_per_room ~ ., lgbm_spec)\n\nlgbm_param <- \n lgbm_wflow |>\n extract_parameter_set_dials() |>\n update(learn_rate = learn_rate(c(-5, -1)))" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#where-are-the-fitted-models", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#where-are-the-fitted-models", - "title": "4 - Evaluating models", - "section": "Where are the fitted models? ", - "text": "Where are the fitted models? \n\ntree_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [48 × 4]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [48 × 4]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [48 × 4]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [47 × 4]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [45 × 4]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n\n\n🗑️" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iterative-search", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iterative-search", + "title": "5 - Iterative Search", + "section": "Iterative Search", + "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our boosted tree.\n\nWe could do something like:\nmae_pred <- lm(mae ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping", - "title": "4 - Evaluating models", - "section": "Bootstrapping", - "text": "Bootstrapping" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iterative-search-1", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iterative-search-1", + "title": "5 - Iterative Search", + "section": "Iterative Search", + "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between MAE and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping-1", - "title": "4 - Evaluating models", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(frog_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [456/163]> Bootstrap01\n#> 2 <split [456/166]> Bootstrap02\n#> 3 <split [456/173]> Bootstrap03\n#> 4 <split [456/177]> Bootstrap04\n#> 5 <split [456/166]> Bootstrap05\n#> 6 <split [456/163]> Bootstrap06\n#> 7 <split [456/164]> Bootstrap07\n#> 8 <split [456/165]> Bootstrap08\n#> 9 <split [456/170]> Bootstrap09\n#> 10 <split [456/177]> Bootstrap10\n#> # ℹ 15 more rows" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#a-large-grid", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#a-large-grid", + "title": "5 - Iterative Search", + "section": "A Large Grid", + "text": "A Large Grid" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-2", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCreate:\n\nbootstrap folds (change times from the default)\nvalidation set (use the reference guide to find the function)\n\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#a-three-point-grid", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#a-three-point-grid", + "title": "5 - Iterative Search", + "section": "A Three Point Grid", + "text": "A Three Point Grid" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#bootstrapping-2", - "title": "4 - Evaluating models", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(322)\nbootstraps(frog_train, times = 10)\n#> # Bootstrap sampling \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [456/173]> Bootstrap01\n#> 2 <split [456/168]> Bootstrap02\n#> 3 <split [456/170]> Bootstrap03\n#> 4 <split [456/164]> Bootstrap04\n#> 5 <split [456/176]> Bootstrap05\n#> 6 <split [456/156]> Bootstrap06\n#> 7 <split [456/166]> Bootstrap07\n#> 8 <split [456/168]> Bootstrap08\n#> 9 <split [456/167]> Bootstrap09\n#> 10 <split [456/170]> Bootstrap10" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gaussian-processes-and-optimization", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gaussian-processes-and-optimization", + "title": "5 - Iterative Search", + "section": "Gaussian Processes and Optimization", + "text": "Gaussian Processes and Optimization\nWe can make a “meta-model” with a small set of historical performance results.\nGaussian Processes (GP) models are a good choice to model performance.\n\nIt is a Bayesian model so we are using Bayesian Optimization (BO).\nFor regression, we can assume that our data are multivariate normal.\nWe also define a covariance function for the variance relationship between data points. A common one is:\n\n\\[\\operatorname{cov}(\\boldsymbol{x}_i, \\boldsymbol{x}_j) = \\exp\\left(-\\frac{1}{2}|\\boldsymbol{x}_i - \\boldsymbol{x}_j|^2\\right) + \\sigma^2_{ij}\\]\n\nGPs are good because\n\nthey are flexible regression models (in the sense that splines are flexible).\nwe need to get mean and variance predictions (and they are Bayesian)\ntheir variability is based on spatial distances.\n\nSome people use random forests (with conformal variance estimates) or other methods but GPs are most popular." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#validation-set", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#validation-set", - "title": "4 - Evaluating models", - "section": "Validation set ", - "text": "Validation set \n\nset.seed(853)\nvalidation_split(frog_train, strata = latency)\n#> # Validation Set Split (0.75/0.25) using stratification \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [340/116]> validation\n\n\nA validation set is just another type of resample" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#predicting-candidates", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#predicting-candidates", + "title": "5 - Iterative Search", + "section": "Predicting Candidates", + "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. MAE)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#random-forest-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#random-forest-1", - "title": "4 - Evaluating models", - "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this tomorrow!), as long as there are enough trees" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#your-turn", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#your-turn", + "title": "5 - Iterative Search", + "section": "Your turn", + "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize MAE.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#create-a-random-forest-model", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#create-a-random-forest-model", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"regression\")\nrf_spec\n#> Random Forest Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", + "title": "5 - Iterative Search", + "section": "GP Fit (ribbon is mean +/- 1SD)", + "text": "GP Fit (ribbon is mean +/- 1SD)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#create-a-random-forest-model-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#create-a-random-forest-model-1", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_wflow <- workflow(latency ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#choosing-new-candidates", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#choosing-new-candidates", + "title": "5 - Iterative Search", + "section": "Choosing New Candidates", + "text": "Choosing New Candidates\nThis isn’t a very good fit but we can still use it.\nHow can we use the outputs to choose the next point to measure?\n\nAcquisition functions take the predicted mean and variance and use them to balance:\n\nexploration: new candidates should explore new areas.\nexploitation: new candidates must stay near existing values.\n\nExploration focuses on the variance, exploitation is about the mean." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-3", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-3", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\nplot true vs. predicted values\n\n\n\n\n−+\n08:00" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#acquisition-functions", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#acquisition-functions", + "title": "5 - Iterative Search", + "section": "Acquisition Functions", + "text": "Acquisition Functions\nWe’ll use an acquisition function to select a new candidate.\nThe most popular method appears to be expected improvement (EI) above the current best results.\n\nZero at existing data points.\nThe expected improvement is integrated over all possible improvement (“expected” in the probability sense).\n\nWe would probably pick the point with the largest EI as the next point.\n(There are other functions beyond EI.)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluating-model-performance-2", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nctrl_frog <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, frog_folds, control = ctrl_frog)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 rmse standard 55.7 10 1.77 pre0_mod0_post0\n#> 2 rsq standard 0.375 10 0.0311 pre0_mod0_post0" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement", + "title": "5 - Iterative Search", + "section": "Expected Improvement", + "text": "Expected Improvement" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-5", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-5", - "title": "4 - Evaluating models", - "section": "", - "text": "collect_predictions(rf_res) %>% \n ggplot(aes(latency, .pred, color = id)) + \n geom_abline(lty = 2, col = \"gray\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iteration", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iteration", + "title": "5 - Iterative Search", + "section": "Iteration", + "text": "Iteration\nOnce we pick the candidate point, we measure performance for it (e.g. resampling).\n\nAnother GP is fit, EI is recomputed, and so on.\n\nWe stop when we have completed the allowed number of iterations or if we don’t see any improvement after a pre-set number of attempts." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", - "title": "4 - Evaluating models", - "section": "How can we compare multiple model workflows at once?", - "text": "How can we compare multiple model workflows at once?" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-fit-with-four-points", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-fit-with-four-points", + "title": "5 - Iterative Search", + "section": "GP Fit with four points", + "text": "GP Fit with four points" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set", - "title": "4 - Evaluating models", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec))\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement-1", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement-1", + "title": "5 - Iterative Search", + "section": "Expected Improvement", + "text": "Expected Improvement" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set-1", - "title": "4 - Evaluating models", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = frog_folds)\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-evolution", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-evolution", + "title": "5 - Iterative Search", + "section": "GP Evolution", + "text": "GP Evolution" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#evaluate-a-workflow-set-2", - "title": "4 - Evaluating models", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = frog_folds) %>%\n rank_results()\n#> # A tibble: 4 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_fo… pre0_m… rmse 55.7 1.82 10 formula rand… 1\n#> 2 formula_rand_fo… pre0_m… rsq 0.375 0.0314 10 formula rand… 1\n#> 3 formula_decisio… pre0_m… rmse 59.6 2.31 10 formula deci… 2\n#> 4 formula_decisio… pre0_m… rsq 0.305 0.0342 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement-evolution", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement-evolution", + "title": "5 - Iterative Search", + "section": "Expected Improvement Evolution", + "text": "Expected Improvement Evolution" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-4", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-4", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nWhen do you think a workflow set would be useful?\n\n\n\n−+\n03:00" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#bo-in-tidymodels", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#bo-in-tidymodels", + "title": "5 - Iterative Search", + "section": "BO in tidymodels", + "text": "BO in tidymodels\nWe’ll use a function called tune_bayes() that has very similar syntax to tune_grid().\n\nIt has an additional initial argument for the initial set of performance estimates and parameter combinations for the GP model." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#the-final-fit", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#the-final-fit", - "title": "4 - Evaluating models", - "section": "The final fit ", - "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# frog_split has train + test info\nfinal_fit <- last_fit(rf_wflow, frog_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [456/116]> train/test split <tibble> <tibble> <tibble> <workflow>" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#initial-grid-points", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#initial-grid-points", + "title": "5 - Iterative Search", + "section": "Initial grid points", + "text": "Initial grid points\ninitial can be the results of another tune_*() function or an integer (in which case tune_grid() is used under to hood to make such an initial set of results).\n\nWe’ll run the optimization more than once, so let’s make an initial grid of results to serve as the substrate for the BO.\nI suggest at least the number of tuning parameters plus two as the initial grid for BO." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 rmse standard 57.4 pre0_mod0_post0\n#> 2 rsq standard 0.414 pre0_mod0_post0\n\n\nThese are metrics computed with the test set" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#an-initial-grid", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#an-initial-grid", + "title": "5 - Iterative Search", + "section": "An Initial Grid", + "text": "An Initial Grid\n\nreg_metrics <- metric_set(mae, rsq)\n\nset.seed(12)\ninit_res <-\n lgbm_wflow |>\n tune_grid(\n resamples = hotel_rs,\n grid = nrow(lgbm_param) + 2,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(init_res, metric = \"mae\") |> select(-.metric, -.estimator)\n#> # A tibble: 5 × 9\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config \n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> \n#> 1 33 12 0.0215 8.25e- 9 5 10.1 10 0.182 Preprocessor1_Model6\n#> 2 14 10 0.1 3.16e+ 1 8 10.2 10 0.192 Preprocessor1_Model3\n#> 3 8 3 0.00464 1 e-10 11 14.4 10 0.305 Preprocessor1_Model2\n#> 4 40 1 0.001 4.64e- 3 14 36.1 10 0.408 Preprocessor1_Model7\n#> 5 27 15 0.000215 6.81e- 7 20 44.5 10 0.371 Preprocessor1_Model5" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit-1", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 116 × 5\n#> .pred id latency .row .config \n#> <dbl> <chr> <dbl> <int> <chr> \n#> 1 44.0 train/test split 22 1 pre0_mod0_post0\n#> 2 101. train/test split 106 3 pre0_mod0_post0\n#> 3 76.0 train/test split 39 6 pre0_mod0_post0\n#> 4 43.1 train/test split 50 8 pre0_mod0_post0\n#> 5 43.5 train/test split 63 10 pre0_mod0_post0\n#> 6 43.5 train/test split 25 14 pre0_mod0_post0\n#> 7 51.4 train/test split 48 16 pre0_mod0_post0\n#> 8 161. train/test split 91 17 pre0_mod0_post0\n#> 9 50.6 train/test split 11 32 pre0_mod0_post0\n#> 10 178. train/test split 109 33 pre0_mod0_post0\n#> # ℹ 106 more rows\n\n\nThese are predictions for the test set" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#bo-using-tidymodels", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#bo-using-tidymodels", + "title": "5 - Iterative Search", + "section": "BO using tidymodels", + "text": "BO using tidymodels\n\nctrl_bo <- control_bayes(verbose_iter = TRUE) # <- for demonstration\n\nset.seed(15)\nlgbm_bayes_res <-\n lgbm_wflow |>\n tune_bayes(\n resamples = hotel_rs,\n initial = init_res, # <- initial results\n iter = 20,\n param_info = lgbm_param,\n control = ctrl_bo,\n metrics = reg_metrics\n )\n#> Optimizing mae using the expected improvement\n#> \n#> ── Iteration 1 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=10.05 (@iter 0)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=7, tree_depth=13, learn_rate=0.0802, loss_reduction=1.13e-10, stop_iter=17\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.599 (+/-0.135)\n#> \n#> ── Iteration 2 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=7, learn_rate=0.0255, loss_reduction=1.22e-07, stop_iter=14\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.895 (+/-0.183)\n#> \n#> ── Iteration 3 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=28, tree_depth=5, learn_rate=0.0507, loss_reduction=1.91e-09, stop_iter=16\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.975 (+/-0.186)\n#> \n#> ── Iteration 4 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=3, tree_depth=11, learn_rate=0.0117, loss_reduction=0.00846, stop_iter=18\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.25 (+/-0.194)\n#> \n#> ── Iteration 5 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=34, tree_depth=1, learn_rate=0.0695, loss_reduction=0.00028, stop_iter=10\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=15.57 (+/-0.254)\n#> \n#> ── Iteration 6 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=18, tree_depth=13, learn_rate=0.0229, loss_reduction=0.181, stop_iter=19\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.882 (+/-0.166)\n#> \n#> ── Iteration 7 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=39, tree_depth=11, learn_rate=0.0485, loss_reduction=0.000906, stop_iter=19\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.883 (+/-0.169)\n#> \n#> ── Iteration 8 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=27, tree_depth=11, learn_rate=0.0396, loss_reduction=1.54e-10, stop_iter=6\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.811 (+/-0.195)\n#> \n#> ── Iteration 9 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=39, tree_depth=5, learn_rate=0.0131, loss_reduction=6.45e-05, stop_iter=16\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.55 (+/-0.152)\n#> \n#> ── Iteration 10 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=15, tree_depth=15, learn_rate=0.00677, loss_reduction=5.01e-06, stop_iter=19\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.73 (+/-0.214)\n#> \n#> ── Iteration 11 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=15, tree_depth=15, learn_rate=0.0929, loss_reduction=2.08, stop_iter=18\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.82 (+/-0.141)\n#> ! No improvement for 10 iterations; returning current results." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-6", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#section-6", - "title": "4 - Evaluating models", - "section": "", - "text": "collect_predictions(final_fit) %>%\n ggplot(aes(latency, .pred)) + \n geom_abline(lty = 2, col = \"deeppink4\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#best-results", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#best-results", + "title": "5 - Iterative Search", + "section": "Best results", + "text": "Best results\n\nshow_best(lgbm_bayes_res, metric = \"mae\") |> select(-.metric, -.estimator)\n#> # A tibble: 5 × 10\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config .iter\n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> <int>\n#> 1 7 13 0.0802 1.13e-10 17 9.60 10 0.135 Iter1 1\n#> 2 27 11 0.0396 1.54e-10 6 9.81 10 0.195 Iter8 8\n#> 3 15 15 0.0929 2.08e+ 0 18 9.82 10 0.141 Iter11 11\n#> 4 18 13 0.0229 1.81e- 1 19 9.88 10 0.166 Iter6 6\n#> 5 39 11 0.0485 9.06e- 4 19 9.88 10 0.169 Iter7 7" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#what-is-in-final_fit-2", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1)) \n#> \n#> Type: Regression \n#> Number of trees: 1000 \n#> Sample size: 456 \n#> Number of independent variables: 4 \n#> Mtry: 2 \n#> Target node size: 5 \n#> Variable importance mode: none \n#> Splitrule: variance \n#> OOB prediction error (MSE): 3099.566 \n#> R squared (OOB): 0.3583602\n\n\nUse this for prediction on new data, like for deploying" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results", + "title": "5 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\")" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-5", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#your-turn-5", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nEnd of the day discussion!\nWhich model do you think you would decide to use?\nWhat surprised you the most?\nWhat is one thing you are looking forward to for tomorrow?\n\n\n\n−+\n05:00" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results-1", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results-1", + "title": "5 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"parameters\")" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit", - "title": "4 - Evaluating models", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results-2", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results-2", + "title": "5 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\")" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-1", - "title": "4 - Evaluating models", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#enhance", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#enhance", + "title": "5 - Iterative Search", + "section": "ENHANCE", + "text": "ENHANCE\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\") +\n ylim(c(9, 14))" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-2", - "title": "4 - Evaluating models", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#your-turn-1", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#your-turn-1", + "title": "5 - Iterative Search", + "section": "Your turn", + "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\nBonus points: Before the optimization is done, press <esc> and see what happens.\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-3", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-3", - "title": "4 - Evaluating models", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#notes", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#notes", + "title": "5 - Iterative Search", + "section": "Notes", + "text": "Notes\n\nStopping tune_bayes() will return the current results.\nParallel processing can still be used to more efficiently measure each candidate point.\nThere are a lot of other iterative methods that you can use.\nThe finetune package also has functions for simulated annealing search." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-4", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-4", - "title": "4 - Evaluating models", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#finalizing-the-model", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#finalizing-the-model", + "title": "5 - Iterative Search", + "section": "Finalizing the Model", + "text": "Finalizing the Model\nLet’s say that we’ve tried a lot of different models and we like our lightgbm model the most.\nWhat do we do now?\n\nFinalize the workflow by choosing the values for the tuning parameters.\nFit the model on the entire training set.\nVerify performance using the test set.\nDocument and publish the model(?)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-5", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#why-choose-just-one-final_fit-5", - "title": "4 - Evaluating models", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#locking-down-the-tuning-parameters", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#locking-down-the-tuning-parameters", + "title": "5 - Iterative Search", + "section": "Locking Down the Tuning Parameters", + "text": "Locking Down the Tuning Parameters\nWe can take the results of the Bayesian optimization and accept the best results:\n\nbest_param <- select_best(lgbm_bayes_res, metric = \"mae\")\nfinal_wflow <- \n lgbm_wflow |> \n finalize_workflow(best_param)\nfinal_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: boost_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> avg_price_per_room ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Boosted Tree Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> min_n = 7\n#> tree_depth = 13\n#> learn_rate = 0.0801616272924067\n#> loss_reduction = 1.13369097175613e-10\n#> stop_iter = 17\n#> \n#> Engine-Specific Arguments:\n#> num_threads = 1\n#> \n#> Computational engine: lightgbm" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nIteratively add candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#the-final-fit", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#the-final-fit", + "title": "5 - Iterative Search", + "section": "The Final Fit", + "text": "The Final Fit\nWe can use individual functions:\nfinal_fit <- final_wflow |> fit(data = hotel_train)\n\n# then predict() or augment() \n# then compute metrics\n\nRemember that there is also a convenience function to do all of this:\n\nset.seed(3893)\nfinal_res <- final_wflow |> last_fit(hotel_split, metrics = reg_metrics)\nfinal_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [3749/1251]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [1,251 × 4]> <workflow>" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-1", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-1", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nstack_ctrl <- control_resamples(save_pred = TRUE, save_workflow = TRUE)" + "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#test-set-results", + "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#test-set-results", + "title": "5 - Iterative Search", + "section": "Test Set Results", + "text": "Test Set Results\n\n\n\nfinal_res |> \n collect_predictions() |> \n cal_plot_regression(\n truth = avg_price_per_room, \n estimate = .pred)\n\nTest set performance:\n\nfinal_res |> collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 9.68 Preprocessor1_Model1\n#> 2 rsq standard 0.946 Preprocessor1_Model1\n\n\n\n\n\n\n\n\n\n\n\n\nRecall that resampling predicted the MAE to be 9.599." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-2", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-2", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nDefine candidate members\n\nStart out with a linear regression:\n\nlr_res <- \n # define model spec\n linear_reg() %>%\n set_mode(\"regression\") %>%\n # add to workflow\n workflow(preprocessor = latency ~ .) %>%\n # fit to resamples\n fit_resamples(frog_folds, control = stack_ctrl)" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---setup", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---setup", + "title": "3 - Tuning Hyperparameters", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-3", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-3", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nlr_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [48 × 4]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [48 × 4]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [48 × 4]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [47 × 4]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [45 × 4]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---data-usage", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---data-usage", + "title": "3 - Tuning Hyperparameters", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-4", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-4", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \nThen, a random forest:\n\nrf_res <- \n # define model spec\n rand_forest() %>%\n set_mode(\"regression\") %>%\n # add to workflow\n workflow(preprocessor = latency ~ .) %>%\n # fit to resamples\n fit_resamples(frog_folds, control = stack_ctrl)" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", + "title": "3 - Tuning Hyperparameters", + "section": "Previously - Feature engineering ", + "text": "Previously - Feature engineering \n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) |>\n step_dummy_hash(company) |>\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors())" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-5", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-5", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nrf_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [48 × 4]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [48 × 4]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [48 × 4]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [47 × 4]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [45 × 4]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 4]> <tibble [44 × 4]>" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-parameters", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-6", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-6", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nInitialize a data stack object\n\n\n\nfrog_st <- stacks()\n\nfrog_st\n#> # A data stack with 0 model definitions and 0 candidate members." + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-7", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-7", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nIteratively add candidate ensemble members to the data stack\n\n\nfrog_st <- frog_st %>%\n add_candidates(lr_res) %>%\n add_candidates(rf_res)\n\nfrog_st\n#> # A data stack with 2 model definitions and 2 candidate members:\n#> # lr_res: 1 model configuration\n#> # rf_res: 1 model configuration\n#> # Outcome: latency (numeric)\n\nTomorrow we’ll discuss tuning parameters where there are different configurations of models (e.g. 10 different variations of the random forest model).\nThese configurations can greatly improve the performance of the stacking ensemble." + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", + "title": "3 - Tuning Hyperparameters", + "section": "Tagging parameters for tuning ", + "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-8", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-8", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nEvaluate how to combine their predictions\n\n\nfrog_st_res <- frog_st %>%\n blend_predictions()\n\nfrog_st_res\n#> # A tibble: 2 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 rf_respre0_mod0_post0 rand_forest 0.650\n#> 2 lr_respre0_mod0_post0 linear_reg 0.325" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", + "title": "3 - Tuning Hyperparameters", + "section": "Optimizing the hash features ", + "text": "Optimizing the hash features \nOur new recipe is:\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) |>\n step_dummy_hash(company, num_terms = tune(\"company hash\")) |>\n step_zv(all_predictors())\n\n\nWe will be using a tree-based model in a minute.\n\nThe other categorical predictors are left as-is.\nThat’s why there is no step_dummy()." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-9", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-9", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nFit candidate ensemble members with non-zero stacking coefficients\n\n\nfrog_st_res <- frog_st_res %>%\n fit_members()\n\nfrog_st_res\n#> # A tibble: 2 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 rf_respre0_mod0_post0 rand_forest 0.650\n#> 2 lr_respre0_mod0_post0 linear_reg 0.325" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-trees", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-trees", + "title": "3 - Tuning Hyperparameters", + "section": "Boosted Trees", + "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved and new samples are predicted using a weighted average of the votes of each tree in the ensemble.\n\nWe’ll focus on the popular lightgbm implementation." }, { - "objectID": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-10", - "href": "archive/2022-07-RStudio-conf/04-evaluating-models.html#building-a-model-stack-10", - "title": "4 - Evaluating models", - "section": "Building a model stack ", - "text": "Building a model stack \n\nPredict on new data!\n\n\n\nfrog_test %>%\n select(latency) %>%\n bind_cols(\n predict(frog_st_res, frog_test)\n ) %>%\n ggplot(aes(latency, .pred)) + \n geom_abline(lty = 2, \n col = \"deeppink4\", \n size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands)\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-parameters", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-parameters", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", + "title": "3 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nTBH it is usually not difficult to optimize these models.\n\nOften, there are multiple candidate tuning parameter combinations that have very good results.\n\nTo demonstrate simple concepts, we’ll look at optimizing the number of trees in the ensemble (between 1 and 100) and the learning rate (\\(10^{-5}\\) to \\(10^{-1}\\))." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#optimize-tuning-parameters", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#optimize-tuning-parameters", - "title": "6 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", + "title": "3 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters ", + "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "title": "3 - Tuning Hyperparameters", "section": "Optimize tuning parameters", "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choosing-tuning-parameters", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choosing-tuning-parameters", - "title": "6 - Tuning Hyperparameters", - "section": "Choosing tuning parameters ", - "text": "Choosing tuning parameters \nLet’s take our previous recipe and add a few changes:\n\nglm_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_lencode_mixed(player, outcome = vars(on_goal)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_mutate(\n angle = abs(atan2(abs(coord_y), (89 - abs(coord_x))) * (180 / pi)),\n distance = sqrt((89 - abs(coord_x))^2 + abs(coord_y)^2),\n distance = log(distance),\n behind_goal_line = ifelse(abs(coord_x) >= 89, 1, 0)\n ) %>%\n step_rm(coord_x, coord_y) %>%\n step_zv(all_predictors()) %>%\n step_ns(angle, deg_free = tune(\"angle\")) %>%\n step_ns(distance, deg_free = tune(\"distance\")) %>%\n step_normalize(all_numeric_predictors())\n\n\nLet’s tune() our spline terms!" - }, - { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choosing-tuning-parameters-1", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choosing-tuning-parameters-1", - "title": "6 - Tuning Hyperparameters", - "section": "Choosing tuning parameters ", - "text": "Choosing tuning parameters \nLet’s take our previous recipe and add a few changes:\n\nglm_spline_wflow <-\n workflow() %>%\n add_model(logistic_reg()) %>%\n add_recipe(glm_rec)" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search", + "title": "3 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#section", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#section", - "title": "6 - Tuning Hyperparameters", - "section": "", - "text": "Splines replace the existing numeric predictor with a set of columns that allow a model to emulate a flexible, nonlinear relationship.\nMore spline terms = more “wiggly”, i.e. flexibly model a nonlinear relationship\nHow many spline terms? This is called degrees of freedom\n2 and 5 look like they underfit; 20 and 100 look like they overfit" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-1", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-1", + "title": "3 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nIn reality we would probably sample the space more densely:" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#splines-and-nonlinear-relationships", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#splines-and-nonlinear-relationships", - "title": "6 - Tuning Hyperparameters", - "section": "Splines and nonlinear relationships", - "text": "Splines and nonlinear relationships\n\n\nOur hockey data exhibits nonlinear relationships\nWe can model nonlinearity like this via a model (later this afternoon) or feature engineering\nHow do we decide how “wiggly” or flexible to make our spline features? TUNING" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#iterative-search", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#iterative-search", + "title": "3 - Tuning Hyperparameters", + "section": "Iterative Search", + "text": "Iterative Search\nWe could start with a few points and search the space:" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#grid-search", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#grid-search", - "title": "6 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nParameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#parameters", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Parameters", + "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#different-types-of-grids", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#different-types-of-grids", + "title": "3 - Tuning Hyperparameters", + "section": "Different types of grids ", + "text": "Different types of grids \n\n\n\n\n\n\n\n\n\nSpace-filling designs (SFD) attempt to cover the parameter space without redundant candidates. We recommend these the most." + }, + { + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-grid", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-grid", + "title": "3 - Tuning Hyperparameters", "section": "Create a grid ", - "text": "Create a grid \n\nglm_spline_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 2 parameters for tuning\n#> \n#> identifier type object\n#> angle deg_free nparam[+]\n#> distance deg_free nparam[+]\n\n\nA parameter set can be updated (e.g. to change the ranges)." + "text": "Create a grid \n\nlgbm_wflow |> \n extract_parameter_set_dials()\n\n# Individual functions: \ntrees()\nlearn_rate()\n\n\nA parameter set can be updated (e.g. to change the ranges)." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid-1", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid-1", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-grid-1", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-grid-1", + "title": "3 - Tuning Hyperparameters", "section": "Create a grid ", - "text": "Create a grid \n\n\n\nset.seed(2)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 23 × 2\n#> angle distance\n#> <int> <int>\n#> 1 12 4\n#> 2 15 8\n#> 3 6 14\n#> 4 10 5\n#> 5 12 12\n#> 6 7 8\n#> 7 14 3\n#> 8 14 13\n#> 9 11 12\n#> 10 8 11\n#> # … with 13 more rows\n#> # ℹ Use `print(n = ...)` to see more rows\n\n\n\n\nA space-filling design like this tends to perform better than random grids.\nSpace-filling designs are also usually more efficient than regular grids." + "text": "Create a grid \n\ngrid <- \n lgbm_wflow |> \n extract_parameter_set_dials() |> \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 7.50e- 6 574 574\n#> 2 84 1.78e- 5 2048 2298\n#> 3 167 5.62e-10 1824 912\n#> 4 250 4.22e- 5 3250 512\n#> 5 334 1.78e- 8 512 2896\n#> 6 417 1.33e- 3 322 1625\n#> 7 500 1 e- 1 1448 1149\n#> 8 584 1 e- 7 1290 256\n#> 9 667 2.37e-10 456 724\n#> 10 750 1.78e- 2 645 322\n#> # ℹ 15 more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn", + "title": "3 - Tuning Hyperparameters", "section": "Your turn", - "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n03:00" + "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid-2", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#create-a-grid-2", - "title": "6 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\nset.seed(2)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 25)\n\ngrid\n#> # A tibble: 225 × 2\n#> angle distance\n#> <int> <int>\n#> 1 1 1\n#> 2 2 1\n#> 3 3 1\n#> 4 4 1\n#> 5 5 1\n#> 6 6 1\n#> 7 7 1\n#> 8 8 1\n#> 9 9 1\n#> 10 10 1\n#> # … with 215 more rows\n#> # ℹ Use `print(n = ...)` to see more rows\n\n\nNote that even though we requested 25x25=625 rows, we only got 15x15=225 back, since the deg_free parameters only have a range of 1->15." + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", + "title": "3 - Tuning Hyperparameters", + "section": "Create a regular grid ", + "text": "Create a regular grid \n\ngrid <- \n lgbm_wflow |> \n extract_parameter_set_dials() |> \n grid_regular(levels = 4)\n\ngrid\n#> # A tibble: 256 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.0000000001 256 256\n#> 2 667 0.0000000001 256 256\n#> 3 1333 0.0000000001 256 256\n#> 4 2000 0.0000000001 256 256\n#> 5 1 0.0000001 256 256\n#> 6 667 0.0000001 256 256\n#> 7 1333 0.0000001 256 256\n#> 8 2000 0.0000001 256 256\n#> 9 1 0.0001 256 256\n#> 10 667 0.0001 256 256\n#> # ℹ 246 more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#update-parameter-ranges", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#update-parameter-ranges", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn-1", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn-1", + "title": "3 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\n\nWhat advantage would a regular grid have?" + }, + { + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", + "title": "3 - Tuning Hyperparameters", "section": "Update parameter ranges ", - "text": "Update parameter ranges \n\nset.seed(2)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n update(angle = spline_degree(c(2L, 20L)),\n distance = spline_degree(c(2L, 20L))) %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 24 × 2\n#> angle distance\n#> <int> <int>\n#> 1 16 6\n#> 2 20 11\n#> 3 8 19\n#> 4 14 7\n#> 5 16 17\n#> 6 10 11\n#> 7 19 5\n#> 8 18 17\n#> 9 15 16\n#> 10 11 15\n#> # … with 14 more rows\n#> # ℹ Use `print(n = ...)` to see more rows\n\n\nEven though angle is a deg_free parameter in step_ns(), we don’t use the dials deg_free() object here. We have a special spline_degree() function that has better defaults for splines." + "text": "Update parameter ranges \n\nlgbm_param <- \n lgbm_wflow |> \n extract_parameter_set_dials() |> \n update(trees = trees(c(1L, 100L)),\n learn_rate = learn_rate(c(-5, -1)))\n\ngrid <- \n lgbm_param |> \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.00147 574 574\n#> 2 5 0.00215 2048 2298\n#> 3 9 0.0000215 1824 912\n#> 4 13 0.00316 3250 512\n#> 5 17 0.0001 512 2896\n#> 6 21 0.0147 322 1625\n#> 7 25 0.1 1448 1149\n#> 8 29 0.000215 1290 256\n#> 9 34 0.0000147 456 724\n#> 10 38 0.0464 645 322\n#> # ℹ 15 more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#the-results", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#the-results", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#the-results", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#the-results", + "title": "3 - Tuning Hyperparameters", "section": "The results ", - "text": "The results \n\n\ngrid %>% \n ggplot(aes(angle, distance)) +\n geom_point(size = 4)" + "text": "The results \n\n\ngrid |> \n ggplot(aes(trees, learn_rate)) +\n geom_point(size = 4) +\n scale_y_log10()\n\n\n\n\n\n\n\n\n\nNote that the learning rates are uniform on the log-10 scale and this shows 2 of 4 dimensions." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#spline-grid-search", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#spline-grid-search", - "title": "6 - Tuning Hyperparameters", - "section": "Spline grid search ", - "text": "Spline grid search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE, parallel_over = \"everything\")\n\nglm_spline_res <-\n glm_spline_wflow %>%\n tune_grid(resamples = nhl_val, grid = grid, control = ctrl)\n\nglm_spline_res\n#> # Tuning results\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7288/1822]> validation <tibble [48 × 6]> <tibble [24 × 3]> <tibble [43,728 × 8]>\n#> \n#> There were issues with some computations:\n#> \n#> - Warning(s) x24: prediction from a rank-deficient fit may be misleading\n#> \n#> Run `show_notes(.Last.tune.result)` for more information.\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Choosing tuning parameters ", + "text": "Choosing tuning parameters \nLet’s take our previous model and tune more parameters:\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)\n\n# Update the feature hash ranges (log-2 units)\nlgbm_param <-\n lgbm_wflow |>\n extract_parameter_set_dials() |>\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-1", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-1", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nTune our glm_wflow.\nWhat happens if you don’t supply a grid argument to tune_grid()?\n\n\n\n05:00" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-3", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-3", + "title": "3 - Tuning Hyperparameters", + "section": "Grid Search ", + "text": "Grid Search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE)\n\nlgbm_res <-\n lgbm_wflow |>\n tune_grid(\n resamples = hotel_rs,\n grid = 25,\n # The options below are not required by default\n param_info = lgbm_param, \n control = ctrl,\n metrics = reg_metrics\n )\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#grid-results", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#grid-results", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-4", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-4", + "title": "3 - Tuning Hyperparameters", + "section": "Grid Search ", + "text": "Grid Search \n\nlgbm_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [3372/377]> Fold01 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,425 × 9]>\n#> 2 <split [3373/376]> Fold02 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 3 <split [3373/376]> Fold03 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 4 <split [3373/376]> Fold04 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 5 <split [3373/376]> Fold05 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 6 <split [3374/375]> Fold06 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,375 × 9]>\n#> 7 <split [3375/374]> Fold07 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,350 × 9]>\n#> 8 <split [3376/373]> Fold08 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 9 <split [3376/373]> Fold09 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 10 <split [3376/373]> Fold10 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>" + }, + { + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-results", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-results", + "title": "3 - Tuning Hyperparameters", "section": "Grid results ", - "text": "Grid results \n\nautoplot(glm_spline_res)" + "text": "Grid results \n\nautoplot(lgbm_res)" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-results", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-results", + "title": "3 - Tuning Hyperparameters", "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(glm_spline_res)\n#> # A tibble: 48 × 8\n#> angle distance .metric .estimator mean n std_err .config \n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 16 6 accuracy binary 0.610 1 NA Preprocessor01_Model1\n#> 2 16 6 roc_auc binary 0.649 1 NA Preprocessor01_Model1\n#> 3 20 11 accuracy binary 0.613 1 NA Preprocessor02_Model1\n#> 4 20 11 roc_auc binary 0.649 1 NA Preprocessor02_Model1\n#> 5 8 19 accuracy binary 0.619 1 NA Preprocessor03_Model1\n#> 6 8 19 roc_auc binary 0.652 1 NA Preprocessor03_Model1\n#> 7 14 7 accuracy binary 0.610 1 NA Preprocessor04_Model1\n#> 8 14 7 roc_auc binary 0.650 1 NA Preprocessor04_Model1\n#> 9 16 17 accuracy binary 0.617 1 NA Preprocessor05_Model1\n#> 10 16 17 roc_auc binary 0.649 1 NA Preprocessor05_Model1\n#> # … with 38 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" + "text": "Tuning results \n\ncollect_metrics(lgbm_res)\n#> # A tibble: 50 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1500 21 0.00000000316 8 60 mae standard 53.2 10 0.427 pre01_mod19_post0\n#> 2 1500 21 0.00000000316 8 60 rsq standard 0.809 10 0.00661 pre01_mod19_post0\n#> 3 917 16 0.0422 9 39 mae standard 9.85 10 0.150 pre02_mod12_post0\n#> 4 917 16 0.0422 9 39 rsq standard 0.946 10 0.00362 pre02_mod12_post0\n#> 5 584 13 0.0000001 10 10 mae standard 53.2 10 0.427 pre03_mod08_post0\n#> 6 584 13 0.0000001 10 10 rsq standard 0.810 10 0.00745 pre03_mod08_post0\n#> 7 167 19 0.00000133 12 143 mae standard 53.2 10 0.426 pre04_mod03_post0\n#> 8 167 19 0.00000133 12 143 rsq standard 0.811 10 0.00698 pre04_mod03_post0\n#> 9 1583 33 0.0000422 14 12 mae standard 50.3 10 0.405 pre05_mod20_post0\n#> 10 1583 33 0.0000422 14 12 rsq standard 0.817 10 0.00768 pre05_mod20_post0\n#> # ℹ 40 more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-1", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-1", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-results-1", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-results-1", + "title": "3 - Tuning Hyperparameters", "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(glm_spline_res, summarize = FALSE)\n#> # A tibble: 48 × 7\n#> id angle distance .metric .estimator .estimate .config \n#> <chr> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 validation 16 6 accuracy binary 0.610 Preprocessor01_Model1\n#> 2 validation 16 6 roc_auc binary 0.649 Preprocessor01_Model1\n#> 3 validation 20 11 accuracy binary 0.613 Preprocessor02_Model1\n#> 4 validation 20 11 roc_auc binary 0.649 Preprocessor02_Model1\n#> 5 validation 8 19 accuracy binary 0.619 Preprocessor03_Model1\n#> 6 validation 8 19 roc_auc binary 0.652 Preprocessor03_Model1\n#> 7 validation 14 7 accuracy binary 0.610 Preprocessor04_Model1\n#> 8 validation 14 7 roc_auc binary 0.650 Preprocessor04_Model1\n#> 9 validation 16 17 accuracy binary 0.617 Preprocessor05_Model1\n#> 10 validation 16 17 roc_auc binary 0.649 Preprocessor05_Model1\n#> # … with 38 more rows\n#> # ℹ Use `print(n = ...)` to see more rows" + "text": "Tuning results \n\ncollect_metrics(lgbm_res, summarize = FALSE)\n#> # A tibble: 500 × 10\n#> id trees min_n learn_rate `agent hash` `company hash` .metric .estimator .estimate .config \n#> <chr> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 Fold01 1500 21 0.00000000316 8 60 mae standard 51.8 pre01_mod19_post0\n#> 2 Fold01 1500 21 0.00000000316 8 60 rsq standard 0.805 pre01_mod19_post0\n#> 3 Fold02 1500 21 0.00000000316 8 60 mae standard 52.1 pre01_mod19_post0\n#> 4 Fold02 1500 21 0.00000000316 8 60 rsq standard 0.800 pre01_mod19_post0\n#> 5 Fold03 1500 21 0.00000000316 8 60 mae standard 52.2 pre01_mod19_post0\n#> 6 Fold03 1500 21 0.00000000316 8 60 rsq standard 0.783 pre01_mod19_post0\n#> 7 Fold04 1500 21 0.00000000316 8 60 mae standard 51.7 pre01_mod19_post0\n#> 8 Fold04 1500 21 0.00000000316 8 60 rsq standard 0.818 pre01_mod19_post0\n#> 9 Fold05 1500 21 0.00000000316 8 60 mae standard 55.2 pre01_mod19_post0\n#> 10 Fold05 1500 21 0.00000000316 8 60 rsq standard 0.845 pre01_mod19_post0\n#> # ℹ 490 more rows" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", + "title": "3 - Tuning Hyperparameters", "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \n\nshow_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 8\n#> angle distance .metric .estimator mean n std_err .config \n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 5 8 roc_auc binary 0.653 1 NA Preprocessor12_Model1\n#> 2 6 9 roc_auc binary 0.653 1 NA Preprocessor17_Model1\n#> 3 3 15 roc_auc binary 0.653 1 NA Preprocessor13_Model1\n#> 4 3 13 roc_auc binary 0.652 1 NA Preprocessor11_Model1\n#> 5 5 19 roc_auc binary 0.652 1 NA Preprocessor24_Model1" + "text": "Choose a parameter combination \n\nshow_best(lgbm_res, metric = \"rsq\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1250 9 0.1 107 80 rsq standard 0.947 10 0.00395 pre19_mod16_post0\n#> 2 917 16 0.0422 9 39 rsq standard 0.946 10 0.00362 pre02_mod12_post0\n#> 3 1666 30 0.00750 124 19 rsq standard 0.943 10 0.00309 pre20_mod21_post0\n#> 4 2000 24 0.00133 25 124 rsq standard 0.919 10 0.00383 pre09_mod25_post0\n#> 5 500 25 0.00316 52 8 rsq standard 0.899 10 0.00435 pre14_mod07_post0" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "title": "6 - Tuning Hyperparameters", + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "title": "3 - Tuning Hyperparameters", "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nselect_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 1 × 3\n#> angle distance .config \n#> <int> <int> <chr> \n#> 1 5 8 Preprocessor12_Model1\n\n\nThis best result has:\n\nlow-degree spline for angle (less “wiggly”, less complex)\nhigher-degree spline for distance (more “wiggly”, more complex)" + "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nlgbm_best <- select_best(lgbm_res, metric = \"mae\")\nlgbm_best\n#> # A tibble: 1 × 6\n#> trees min_n learn_rate `agent hash` `company hash` .config \n#> <int> <int> <dbl> <int> <int> <chr> \n#> 1 1250 9 0.1 107 80 pre19_mod16_post0" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-2", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-2", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nTry an alternative selection strategy.\nRead the docs for select_by_pct_loss().\nTry choosing a model that has a simpler (less “wiggly”) relationship for distance.\n\n\n\n05:00" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#checking-calibration", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#checking-calibration", + "title": "3 - Tuning Hyperparameters", + "section": "Checking Calibration ", + "text": "Checking Calibration \n\n\nlibrary(probably)\nlgbm_res |>\n collect_predictions(\n parameters = lgbm_best\n ) |>\n cal_plot_regression(\n truth = avg_price_per_room,\n estimate = .pred\n )" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination-2", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#choose-a-parameter-combination-2", - "title": "6 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \n\nselect_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 1 × 3\n#> angle distance .config \n#> <int> <int> <chr> \n#> 1 5 8 Preprocessor12_Model1\nselect_by_pct_loss(glm_spline_res, distance, metric = \"roc_auc\")\n#> # A tibble: 1 × 10\n#> angle distance .metric .estimator mean n std_err .config .best .loss\n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> <dbl> <dbl>\n#> 1 13 4 roc_auc binary 0.646 1 NA Preprocessor20_Model1 0.653 0.984" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#running-in-parallel", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#running-in-parallel", + "title": "3 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\n\nlibrary(future)\nplan(multisession, workers = cores)\n\n# Now call `tune_grid()`!" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-trees-1", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-trees-1", - "title": "6 - Tuning Hyperparameters", - "section": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\n\n\nReview how a decision tree model works:\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", + "title": "3 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#single-decision-tree", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#single-decision-tree", - "title": "6 - Tuning Hyperparameters", - "section": "Single decision tree", - "text": "Single decision tree" + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", + "title": "3 - Tuning Hyperparameters", + "section": "The ‘future’ of parallel processing", + "text": "The ‘future’ of parallel processing\nWe have relied on the foreach package for parallel processing.\nWe will start the transition to using the future package in the upcoming version of the tune package (version 1.3.0).\nThere will be a period of backward compatibility where you can still use foreach with future via the doFuture package. After that, the transition to future will occur.\nOverall, there will be minimal changes to your code." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-trees-2", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-trees-2", - "title": "6 - Tuning Hyperparameters", - "section": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\nBoosting methods fit a sequence of tree-based models.\n\n\nEach tree is dependent on the one before and tries to compensate for any poor results in the previous trees.\nThis is like gradient-based steepest ascent methods from calculus." + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", + "title": "3 - Tuning Hyperparameters", + "section": "Early stopping for boosted trees", + "text": "Early stopping for boosted trees\nWe have directly optimized the number of trees as a tuning parameter.\nInstead we could\n\nSet the number of trees to a single large number.\nStop adding trees when performance gets worse.\n\nThis is known as “early stopping” and there is a parameter for that: stop_iter.\nEarly stopping has a potential to decrease the tuning time." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "title": "6 - Tuning Hyperparameters", - "section": "Boosted tree tuning parameters", - "text": "Boosted tree tuning parameters\nMost modern boosting methods have a lot of tuning parameters!\n\n\nFor tree growth and pruning (min_n, max_depth, etc)\nFor boosting (trees, stop_iter, learn_rate)\n\n\n\nWe’ll use early stopping to stop boosting when a few iterations produce consecutively worse results." + "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn-2", + "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn-2", + "title": "3 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\n\nSet trees = 2000 and tune the stop_iter parameter.\nNote that you will need to regenerate lgbm_param with your new workflow!\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#comparing-tree-ensembles", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#comparing-tree-ensembles", - "title": "6 - Tuning Hyperparameters", - "section": "Comparing tree ensembles", - "text": "Comparing tree ensembles\n\n\nRandom forest\n\nIndependent trees\nBootstrapped data\nNo pruning\n1000’s of trees\n\n\nBoosting\n\nDependent trees\nDifferent case weights\nTune tree parameters\nFar fewer trees\n\n\nThe general consensus for tree-based models is, in terms of performance: boosting > random forest > bagging > single trees." + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#who-are-you", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML\nYou have used some tidymodels packages\nYou have some experience with evaluating statistical models using resampling techniques" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-tree-code", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#boosted-tree-code", - "title": "6 - Tuning Hyperparameters", - "section": "Boosted tree code", - "text": "Boosted tree code\n\nxgb_spec <-\n boost_tree(\n trees = 500, min_n = tune(), stop_iter = tune(), tree_depth = tune(),\n learn_rate = tune(), loss_reduction = tune()\n ) %>%\n set_mode(\"classification\") %>% \n set_engine(\"xgboost\", validation = 1/10) # <- for better early stopping\n\nxgb_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_lencode_mixed(player, outcome = vars(on_goal)) %>% \n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\nxgb_wflow <- \n workflow() %>% \n add_model(xgb_spec) %>% \n add_recipe(xgb_rec)\n\n\nvalidation is an argument to parsnip::xgb_train(), not directly to xgboost. It generates a validation set that is used by xgboost when evaluating model performance. It is eventually assigned to xgb.train(watchlist = list(validation = data)).\nSee translate(xgb_spec) to see where it is passed to parsnip::xgb_train()." + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#who-are-tidymodels", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-3", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-3", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nCreate your boosted tree workflow.\n\n\n\n03:00" + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-2", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-2", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#running-in-parallel", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#running-in-parallel", - "title": "6 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallel::detectCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-3", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-3", + "title": "1 - Introduction", + "section": "👀", + "text": "👀\n\n\nhttps://workshops.tidymodels.org" + }, + { + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#tentative-plan-for-this-workshop", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#tentative-plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Tentative plan for this workshop", + "text": "Tentative plan for this workshop\n\nFeature engineering with recipes\nModel optimization by tuning\n\nGrid search\nRacing\nIterative methods\n\nExtras (time permitting)\n\nEffect encodings\nA case study" + }, + { + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-4", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-4", + "title": "1 - Introduction", + "section": "", + "text": "Introduce yourself to your neighbors 👋" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#running-in-parallel-1", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#running-in-parallel-1", - "title": "6 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#lets-install-some-packages", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"lightgbm\", \"lme4\", \"pak\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\npak::pak(\"simonpcouch/forested\")\n\nAlso, you should install the newest version of the dials package (version 1.3.0). To check this, you can run:\n\nrlang::check_installed(\"dials\", version = \"1.3.0\")" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning ", - "text": "Tuning \nThis will take some time to run ⏳\n\nset.seed(9)\n\nxgb_res <-\n xgb_wflow %>%\n tune_grid(resamples = nhl_val, grid = 15, control = ctrl) # automatic grid now!" + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#hotel-data", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#hotel-data", + "title": "1 - Introduction", + "section": "Hotel Data ", + "text": "Hotel Data \nWe’ll use data on hotels to predict the cost of a room.\nThe data are in the modeldata package. We’ll sample down the data and refactor some columns:\n\n\n\nlibrary(tidymodels)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-4", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-4", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nStart tuning the boosted tree model!\nWe won’t wait for everyone’s tuning to finish, but take this time to get it started before we move on.\n\n\n\n03:00" + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#hotel-date-columns", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#hotel-date-columns", + "title": "1 - Introduction", + "section": "Hotel date columns", + "text": "Hotel date columns\n\nnames(hotel_rates)\n#> [1] \"avg_price_per_room\" \"lead_time\" \n#> [3] \"stays_in_weekend_nights\" \"stays_in_week_nights\" \n#> [5] \"adults\" \"children\" \n#> [7] \"babies\" \"meal\" \n#> [9] \"country\" \"market_segment\" \n#> [11] \"distribution_channel\" \"is_repeated_guest\" \n#> [13] \"previous_cancellations\" \"previous_bookings_not_canceled\"\n#> [15] \"reserved_room_type\" \"assigned_room_type\" \n#> [17] \"booking_changes\" \"agent\" \n#> [19] \"company\" \"days_in_waiting_list\" \n#> [21] \"customer_type\" \"required_car_parking_spaces\" \n#> [23] \"total_of_special_requests\" \"arrival_date_num\" \n#> [25] \"near_christmas\" \"near_new_years\" \n#> [27] \"historical_adr\"" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-2", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-2", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\nxgb_res\n#> # Tuning results\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7288/1822]> validation <tibble [30 × 9]> <tibble [0 × 3]> <tibble [27,330 × 11]>" + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#data-splitting-strategy", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#data-splitting-strategy", + "title": "1 - Introduction", + "section": "Data splitting strategy", + "text": "Data splitting strategy" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-3", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#tuning-results-3", - "title": "6 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\nautoplot(xgb_res)" + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#data-spending", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#data-spending", + "title": "1 - Introduction", + "section": "Data Spending ", + "text": "Data Spending \nLet’s split the data into a training set (75%) and testing set (25%) using stratification:\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#again-with-the-location-features", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#again-with-the-location-features", - "title": "6 - Tuning Hyperparameters", - "section": "Again with the location features", - "text": "Again with the location features\n\ncoord_rec <- \n xgb_rec %>%\n step_mutate(\n angle = abs(atan2(abs(coord_y), (89 - abs(coord_x))) * (180 / pi)),\n distance = sqrt((89 - abs(coord_x))^2 + abs(coord_y)^2),\n distance = log(distance),\n behind_goal_line = ifelse(abs(coord_x) >= 89, 1, 0)\n ) %>% \n step_rm(coord_x, coord_y)\n\nxgb_coord_wflow <- \n workflow() %>% \n add_model(xgb_spec) %>% \n add_recipe(coord_rec)\n\nset.seed(9)\nxgb_coord_res <-\n xgb_coord_wflow %>%\n tune_grid(resamples = nhl_val, grid = 20, control = ctrl)" + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#your-turn", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#your-turn", + "title": "1 - Introduction", + "section": "Your turn", + "text": "Your turn\nLet’s take some time and investigate the training data. The outcome is avg_price_per_room.\nAre there any interesting characteristics of the data?\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#did-the-machine-figure-it-out", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#did-the-machine-figure-it-out", - "title": "6 - Tuning Hyperparameters", - "section": "Did the machine figure it out?", - "text": "Did the machine figure it out?\n\nshow_best(xgb_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 11\n#> min_n tree_depth learn_rate loss_reduction stop_iter .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <dbl> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 19 2 0.311 1.46e- 4 14 roc_auc binary 0.633 1 NA Preprocessor1_Model12\n#> 2 11 3 0.0255 5.61e- 7 19 roc_auc binary 0.628 1 NA Preprocessor1_Model05\n#> 3 27 5 0.0120 6.04e- 6 12 roc_auc binary 0.627 1 NA Preprocessor1_Model07\n#> 4 25 14 0.0379 3.62e- 5 8 roc_auc binary 0.626 1 NA Preprocessor1_Model13\n#> 5 31 11 0.00585 1.02e-10 7 roc_auc binary 0.625 1 NA Preprocessor1_Model08\n\nshow_best(xgb_coord_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 11\n#> min_n tree_depth learn_rate loss_reduction stop_iter .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <dbl> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 30 13 0.0578 0.00000000141 18 roc_auc binary 0.648 1 NA Preprocessor1_Model07\n#> 2 39 12 0.0803 0.000411 10 roc_auc binary 0.643 1 NA Preprocessor1_Model12\n#> 3 14 2 0.146 0.00244 19 roc_auc binary 0.642 1 NA Preprocessor1_Model14\n#> 4 26 15 0.0365 2.51 17 roc_auc binary 0.642 1 NA Preprocessor1_Model11\n#> 5 35 5 0.101 0.0000000784 13 roc_auc binary 0.641 1 NA Preprocessor1_Model17" + "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#our-versions", + "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nR version 4.5.0 (2025-04-11), Quarto (1.6.42)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.4.0\n\n\nbroom\n1.0.9\n\n\nCubist\n0.5.0\n\n\ndials\n1.4.1\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.4\n\n\nembed\n1.1.5.9000\n\n\nfinetune\n1.2.1.9000\n\n\nforested\n0.2.0\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nFormula\n1.2-5\n\n\nggplot2\n3.5.2\n\n\nlattice\n0.22-7\n\n\nlightgbm\n4.6.0\n\n\nlme4\n1.1-37\n\n\nmodeldata\n1.5.1\n\n\nparallelly\n1.45.1\n\n\nparsnip\n1.3.2.9000\n\n\nplotmo\n3.6.4\n\n\nplotrix\n3.8-4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nplumber\n1.3.0\n\n\nprobably\n1.1.0\n\n\npurrr\n1.1.0\n\n\nrecipes\n1.3.1\n\n\nrsample\n1.3.1\n\n\nrules\n1.0.2\n\n\nscales\n1.4.0\n\n\nsplines2\n0.5.4\n\n\nstacks\n1.1.1.9000\n\n\ntext2vec\n0.6.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntextrecipes\n1.1.0.9000\n\n\ntibble\n3.3.0\n\n\ntidymodels\n1.3.0\n\n\ntidyr\n1.3.1\n\n\ntune\n1.3.0\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.2.0.9002\n\n\nworkflowsets\n1.1.1\n\n\nyardstick\n1.3.2" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#compare-models", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#compare-models", - "title": "6 - Tuning Hyperparameters", - "section": "Compare models", - "text": "Compare models\nBest logistic regression results:\n\nglm_spline_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, .estimator, mean, n, std_err, .config)\n#> # A tibble: 1 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 roc_auc binary 0.653 1 NA Preprocessor12_Model1\n\n\nBest boosting results:\n\nxgb_coord_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, .estimator, mean, n, std_err, .config)\n#> # A tibble: 1 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 roc_auc binary 0.648 1 NA Preprocessor1_Model07" + "objectID": "index.html", + "href": "index.html", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels offered at posit::conf(2025). The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to Machine Learning in R with tidymodels and Getting More Out of Feature Engineering and Tuning for Machine Learning courses.\nIntroduction to Machine Learning in R with tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nGetting More Out of Feature Engineering and Tuning for Machine Learning will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-5", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-5", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nCan you get better ROC results with xgboost?\nTry increasing learn_rate beyond the original range.\n\n\n\n20:00" + "objectID": "index.html#welcome", + "href": "index.html#welcome", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels offered at posit::conf(2025). The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to Machine Learning in R with tidymodels and Getting More Out of Feature Engineering and Tuning for Machine Learning courses.\nIntroduction to Machine Learning in R with tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nGetting More Out of Feature Engineering and Tuning for Machine Learning will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#updating-the-workflow", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#updating-the-workflow", - "title": "6 - Tuning Hyperparameters", - "section": "Updating the workflow ", - "text": "Updating the workflow \n\nbest_auc <- select_best(glm_spline_res, metric = \"roc_auc\")\nbest_auc\n#> # A tibble: 1 × 3\n#> angle distance .config \n#> <int> <int> <chr> \n#> 1 5 8 Preprocessor12_Model1\n\nglm_spline_wflow <-\n glm_spline_wflow %>% \n finalize_workflow(best_auc)\n\nglm_spline_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: logistic_reg()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 8 Recipe Steps\n#> \n#> • step_lencode_mixed()\n#> • step_dummy()\n#> • step_mutate()\n#> • step_rm()\n#> • step_zv()\n#> • step_ns()\n#> • step_ns()\n#> • step_normalize()\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm" + "objectID": "index.html#is-this-workshop-for-me", + "href": "index.html#is-this-workshop-for-me", + "title": "Machine learning with tidymodels", + "section": "Is this workshop for me? ", + "text": "Is this workshop for me? \nDepending on your background, one of Introduction to Machine Learning in R with tidymodels or Getting More Out of Feature Engineering and Tuning for Machine Learning might serve you better than the other.\n\nIntroduction to Machine Learning in R with tidymodels\nThis workshop is for you if you:\n\nare comfortable using tidyverse packages to read data into R, transform and reshape data, and make a variety of graphs, and\nhave had some exposure to basic statistical concepts such as linear models, residuals, etc.\n\nIntermediate or expert familiarity with modeling or machine learning is not required. Interested students who have intermediate or expert familiarity with modeling or machine learning may be interested in the Getting More Out of Feature Engineering and Tuning for Machine Learning workshop.\n\n\nGetting More Out of Feature Engineering and Tuning for Machine Learning\nThis workshop is for you if you:\n\nhave the prerequisite skills listed for the Introduction to Machine Learning in R with tidymodels workshops,\nhave used tidymodels packages like recipes, rsample, and parsnip, and\nhave some experience with evaluating statistical models using resampling techniques like v-fold cross-validation or the bootstrap.\n\nParticipants who are new to tidymodels or machine learning will benefit from taking the Introduction to Machine Learning in R with tidymodels workshop before joining this one. Participants who have completed the “Introduction to Machine Learning in R with tidymodels” workshop previously will be well-prepared for this course." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#the-final-fit-to-the-nhl-data", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#the-final-fit-to-the-nhl-data", - "title": "6 - Tuning Hyperparameters", - "section": "The final fit to the NHL data ", - "text": "The final fit to the NHL data \n\ntest_res <- \n glm_spline_wflow %>% \n last_fit(split = nhl_split)\n\ntest_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [9110/3037]> train/test split <tibble [2 × 4]> <tibble [1 × 3]> <tibble [3,037 × 6]> <workflow>\n#> \n#> There were issues with some computations:\n#> \n#> - Warning(s) x1: prediction from a rank-deficient fit may be misleading\n#> \n#> Run `show_notes(.Last.tune.result)` for more information.\n\n\nRemember that last_fit() fits one time with the combined training and validation set, then evaluates one time with the testing set." + "objectID": "index.html#preparation", + "href": "index.html#preparation", + "title": "Machine learning with tidymodels", + "section": "Preparation", + "text": "Preparation\nThe process to set up your computer for either workshop will look the same. Please join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of an IDE, either\n\nRStudio Desktop (RStudio Desktop Open Source License, at least v2024.04.0), available at https://posit.co/download/rstudio-desktop/ or\nPositron (Desktop version, at least 2025.08.0-130), available at https://positron.posit.co/download.html\n\nFor all of the slides, the following R packages can be installed from the R console:\n\n\n# Install the packages for the workshop\npkgs <- \n c(\"almanac\", \"betacal\", \"bonsai\", \"brulee\", \"C50\", \"Cubist\", \"desirability2\", \n \"dimRed\", \"embed\", \"extrasteps\", \"finetune\", \"forested\", \n \"igraph\", \"important\", \"irlba\", \"kknn\", \"lightgbm\", \"lme4\", \"mirai\", \n \"parallelly\", \"plumber\", \"probably\", \"ranger\", \"RANN\", \"rpart\", \n \"rpart.plot\", \"RSpectra\", \"rules\", \"splines2\", \"stacks\", \"text2vec\", \n \"textrecipes\", \"tidymodels\", \"uwot\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nAlso, you should make sure that you have installed the newest version of a few packages. To check this, you can run the following code snippet. It will also prompt you to install the required versions if they are not already installed.\n\nrlang::check_installed(\"tidymodels\", version = \"1.4.1\")\nrlang::check_installed(\"embed\", version = \"1.2.0\")\n\nIf you’re a Windows user and encounter an error message during installation noting a missing Rtools installation, install Rtools using the installer linked here." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-6", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-6", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nFinalize your workflow with the best parameters.\nCreate a final fit.\n\n\n\n08:00" + "objectID": "index.html#slides", + "href": "index.html#slides", + "title": "Machine learning with tidymodels", + "section": "Slides", + "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. They are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to Machine Learning in R with tidymodels\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n05: Tuning models\n06: Wrapping up\n\n\n\nGetting More Out of Feature Engineering and Tuning for Machine Learning\n\n01: Introduction\n02: Model optimization by tuning\n03: Grid search via racing\n04: Feature engieering:dummies and embeddings\n05: Feature engineering: splines, target encoding and dates\n06: Postprocessing\n07: Feature selection\n08: Wrapping up\n\n\n\nExtra content (time permitting)\n\nIntro: Using workflowsets\nIntro: Using recipes\nAdvanced: Transit case study (includes stacking)\nAdvanced: Iterative search\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#estimates-of-roc-auc", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#estimates-of-roc-auc", - "title": "6 - Tuning Hyperparameters", - "section": "Estimates of ROC AUC ", - "text": "Estimates of ROC AUC \nValidation results from tuning:\n\nglm_spline_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, mean, n, std_err)\n#> # A tibble: 1 × 4\n#> .metric mean n std_err\n#> <chr> <dbl> <int> <dbl>\n#> 1 roc_auc 0.653 1 NA\n\n\nTest set results:\n\ntest_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.616 Preprocessor1_Model1\n#> 2 roc_auc binary 0.656 Preprocessor1_Model1" + "objectID": "index.html#code", + "href": "index.html#code", + "title": "Machine learning with tidymodels", + "section": "Code", + "text": "Code\nQuarto files for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#final-fitted-workflow", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#final-fitted-workflow", - "title": "6 - Tuning Hyperparameters", - "section": "Final fitted workflow", - "text": "Final fitted workflow\nExtract the final fitted workflow, fit using the training set:\n\nfinal_glm_spline_wflow <- \n test_res %>% \n extract_workflow()\n\n# use this object to predict or deploy\npredict(final_glm_spline_wflow, nhl_test[1:3,])\n#> # A tibble: 3 × 1\n#> .pred_class\n#> <fct> \n#> 1 no \n#> 2 yes \n#> 3 no" + "objectID": "index.html#past-workshops", + "href": "index.html#past-workshops", + "title": "Machine learning with tidymodels", + "section": "Past workshops", + "text": "Past workshops\n\nEnglish\n\nAugust 2025 at New York Data Science & AI Conference\nAugust 2024 at posit::conf()\nSeptember 2023 at posit::conf()\nJuly 2023 at the New York R Conference\nAugust 2022 in Reykjavik\nJuly 2022 at rstudio::conf()\n\n\n\nSpanish\n\nMarch 2024 at conectaR" }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#next-steps", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#next-steps", - "title": "6 - Tuning Hyperparameters", - "section": "Next steps", - "text": "Next steps\n\nDocument the model.\n\n\n\nDeploy the model.\n\n\n\n\nCreate an applicability domain model to help monitor our data over time.\n\n\n\n\nUse explainers to characterize the model and the predictions." + "objectID": "index.html#acknowledgments", + "href": "index.html#acknowledgments", + "title": "Machine learning with tidymodels", + "section": "Acknowledgments", + "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-yourself", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-yourself", - "title": "6 - Tuning Hyperparameters", - "section": "Explain yourself", - "text": "Explain yourself\nThere are two categories of model explanations, global and local.\n\n\nGlobal model explanations provide an overall understanding aggregated over a whole set of observations.\nLocal model explanations provide information about a prediction for a single observation.\n\n\n\nYou can also build global model explanations by aggregating local model explanations." + "objectID": "index.html#reuse-and-licensing", + "href": "index.html#reuse-and-licensing", + "title": "Machine learning with tidymodels", + "section": "Reuse and licensing", + "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#a-tidymodels-explainer", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#a-tidymodels-explainer", - "title": "6 - Tuning Hyperparameters", - "section": "A tidymodels explainer", - "text": "A tidymodels explainer\nWe can build explainers using:\n\noriginal, basic predictors\nderived features\n\n\nlibrary(DALEXtra)\n\nglm_explainer <- explain_tidymodels(\n final_glm_spline_wflow,\n data = dplyr::select(nhl_train, -on_goal),\n # DALEX required an integer for factors:\n y = as.integer(nhl_train$on_goal),\n verbose = FALSE\n)" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#working-with-our-predictors", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#working-with-our-predictors", + "title": "2 - Feature Engineering", + "section": "Working with our predictors", + "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for linear regression).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates", - "title": "6 - Tuning Hyperparameters", - "section": "Explain the x coordinates", - "text": "Explain the x coordinates\nWith our explainer, let’s create partial dependence profiles:\n\nset.seed(123)\npdp_coord_x <- model_profile(\n glm_explainer,\n variables = \"coord_x\",\n N = 500,\n groups = \"position\"\n)\n\n\nYou can use the default plot() method or create your own visualization." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#what-is-feature-engineering", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#what-is-feature-engineering", + "title": "2 - Feature Engineering", + "section": "What is feature engineering?", + "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPrincipal component analysis (PCA) feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection (FES) and Applied Machine Learning for Tabular Data (aml4td)." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates-1", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates-1", - "title": "6 - Tuning Hyperparameters", - "section": "Explain the x coordinates", - "text": "Explain the x coordinates" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#example-dates", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#example-dates", + "title": "2 - Feature Engineering", + "section": "Example: Dates", + "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates-2", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#explain-the-x-coordinates-2", - "title": "6 - Tuning Hyperparameters", - "section": "Explain the x coordinates", - "text": "Explain the x coordinates" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#general-definitions", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#general-definitions", + "title": "2 - Feature Engineering", + "section": "General definitions ", + "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." }, { - "objectID": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-7", - "href": "archive/2022-07-RStudio-conf/06-tuning-hyperparameters.html#your-turn-7", - "title": "6 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nCreate an explainer for our glm model.\nTry grouping by another variable, like game_type or dow.\n\n\n\n05:00" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#previously---setup", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#previously---setup", + "title": "2 - Feature Engineering", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\n\n# Add another package:\nlibrary(textrecipes)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html", - "href": "archive/2022-07-RStudio-conf/annotations.html", - "title": "Annotations", - "section": "", - "text": "This page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources.\n\n\n\nThis is a pretty complex data usage scheme. That is mostly because of the validation set. In every other case, the situation is much more simple.\nThe important point here is that: tidymodels does most of this work for you. In other words, you don’t have to directly specify which data are being used where.\nIn a later section, we will talk about methods of resampling. These methods are like repeated validation sets. As an example, the popular 10-fold cross-validation method is one such type of resampling. Validation sets are special cases of resampling where there is a single “resample”.\nMost types of resampling use multiple hold-out sets of samples from the training set. In those cases, a diagram for data usage here would look like\n\n\n\n\n\n\n\n\n\nIn this case there is just “testing” and “training”. Once the final model is determined, the entire training set is used for the last fit.\nThis is the process that will be used for the tree frog data." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#previously---data-usage", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#previously---data-usage", + "title": "2 - Feature Engineering", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\n\nWe’ll go from here and create a set of resamples to use for model assessments." }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#section", - "href": "archive/2022-07-RStudio-conf/annotations.html#section", - "title": "Annotations", - "section": "", - "text": "This page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#resampling-strategy", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#resampling-strategy", + "title": "2 - Feature Engineering", + "section": "Resampling Strategy", + "text": "Resampling Strategy" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#finalize-and-verify", - "href": "archive/2022-07-RStudio-conf/annotations.html#finalize-and-verify", - "title": "Annotations", - "section": "", - "text": "This is a pretty complex data usage scheme. That is mostly because of the validation set. In every other case, the situation is much more simple.\nThe important point here is that: tidymodels does most of this work for you. In other words, you don’t have to directly specify which data are being used where.\nIn a later section, we will talk about methods of resampling. These methods are like repeated validation sets. As an example, the popular 10-fold cross-validation method is one such type of resampling. Validation sets are special cases of resampling where there is a single “resample”.\nMost types of resampling use multiple hold-out sets of samples from the training set. In those cases, a diagram for data usage here would look like\n\n\n\n\n\n\n\n\n\nIn this case there is just “testing” and “training”. Once the final model is determined, the entire training set is used for the last fit.\nThis is the process that will be used for the tree frog data." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#resampling-strategy-1", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#resampling-strategy-1", + "title": "2 - Feature Engineering", + "section": "Resampling Strategy ", + "text": "Resampling Strategy \nWe’ll use simple 10-fold cross-validation (stratified sampling):\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)\nhotel_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [3372/377]> Fold01\n#> 2 <split [3373/376]> Fold02\n#> 3 <split [3373/376]> Fold03\n#> 4 <split [3373/376]> Fold04\n#> 5 <split [3373/376]> Fold05\n#> 6 <split [3374/375]> Fold06\n#> 7 <split [3375/374]> Fold07\n#> 8 <split [3376/373]> Fold08\n#> 9 <split [3376/373]> Fold09\n#> 10 <split [3376/373]> Fold10" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#data-splitting-and-spending", - "href": "archive/2022-07-RStudio-conf/annotations.html#data-splitting-and-spending", - "title": "Annotations", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n\n#> [1] 0.2655087 0.3721239 0.5728534\n\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", + "title": "2 - Feature Engineering", + "section": "Prepare your data for modeling ", + "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of preprocessing and feature engineering steps.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." + }, + { + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#a-first-recipe", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#a-first-recipe", + "title": "2 - Feature Engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#what-is-wrong-with-this", - "href": "archive/2022-07-RStudio-conf/annotations.html#what-is-wrong-with-this", - "title": "Annotations", - "section": "What is wrong with this?", - "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#a-first-recipe-1", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#a-first-recipe-1", + "title": "2 - Feature Engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nsummary(hotel_rec)\n#> # A tibble: 27 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 lead_time <chr [2]> predictor original\n#> 2 stays_in_weekend_nights <chr [2]> predictor original\n#> 3 stays_in_week_nights <chr [2]> predictor original\n#> 4 adults <chr [2]> predictor original\n#> 5 children <chr [2]> predictor original\n#> 6 babies <chr [2]> predictor original\n#> 7 meal <chr [3]> predictor original\n#> 8 country <chr [3]> predictor original\n#> 9 market_segment <chr [3]> predictor original\n#> 10 distribution_channel <chr [3]> predictor original\n#> # ℹ 17 more rows\n\nThe type column contains information on the variables" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#where-are-the-fitted-models", - "href": "archive/2022-07-RStudio-conf/annotations.html#where-are-the-fitted-models", - "title": "Annotations", - "section": "Where are the fitted models?", - "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn", + "title": "2 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\nWhat do you think are in the type vectors for the lead_time and country columns?\n\n\n\n−+\n02:00" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#the-final-fit", - "href": "archive/2022-07-RStudio-conf/annotations.html#the-final-fit", - "title": "Annotations", - "section": "The final fit", - "text": "The final fit\nSince our data spending scheme created the resamples from the training set, last_fit() will use all of the training data to fit the final workflow.\nAs shown in the Whole Game slides, there is a slightly different scheme used when we have a validation set (instead of multiple resamples like 10-fold CV)." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#create-indicator-variables", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#create-indicator-variables", + "title": "2 - Feature Engineering", + "section": "Create indicator variables ", + "text": "Create indicator variables \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns.\nstep_dummy() records the levels of the categorical predictors in the training set." }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#per-player-statistics", - "href": "archive/2022-07-RStudio-conf/annotations.html#per-player-statistics", - "title": "Annotations", - "section": "Per-player statistics", - "text": "Per-player statistics\nThe effect encoding method essentially takes the effect of a variable, like player, and makes a data column for that effect. In our example, the ability of a player to have an on-goal shot is quantified by a model and then added as a data column to be used in the model.\nSuppose NHL rookie Max has a single shot in the data and it was on goal. If we used a naive estimate for Max’s effect, the model is being told that Max should have a 100% chance of being on goal. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned player Davis, who has taken 250 shots and 75% of these were on goal. Davis’s proportion is more predictive because it is estimated with better data (i.e., more total shots). Partial pooling leverages the entire data set and can borrow strength from all of the players. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a player’s data is of good quality, the partial pooling effect estimate is closer to the raw proportion. Max’s data is not great and is “shrunk” towards the center of the overall on goal proportion. Since there is so little known about Max’s shot history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nIf the outcome were numeric, the effect would be the mean of the outcome per player. In this case, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#filter-out-constant-columns", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#filter-out-constant-columns", + "title": "2 - Feature Engineering", + "section": "Filter out constant columns ", + "text": "Filter out constant columns \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#player-effects", - "href": "archive/2022-07-RStudio-conf/annotations.html#player-effects", - "title": "Annotations", - "section": "Player effects", - "text": "Player effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for players with small sample sizes. It can’t correct for improper data usage or data leakage though." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#normalization", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#normalization", + "title": "2 - Feature Engineering", + "section": "Normalization ", + "text": "Normalization \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#angle", - "href": "archive/2022-07-RStudio-conf/annotations.html#angle", - "title": "Annotations", - "section": "Angle", - "text": "Angle\nAbout these formulas…\nThe coordinates for the rink are centered at (0, 0) and the goal lines are both 89 ft from center. The center of the goal on the left is at (-89, 0) and the right-hand goal is centered at (89, 0).\nFor the distance to center ice, the formula is\n\\[d_{center} = \\sqrt{x^2 + y^2}\\]\nWe want distance to the goal line(s), so we first use x* = (89 - abs(coord_x)) to make the side of the rink irrelevant and then compute\n\\[d_{goal} = \\sqrt{x^{*2} + y^2}\\]\nIn the code, we log the distance value to help its distribution become more symmetric.\nFor angle to center, the formula is\n\\[a = \\tan^{-1}\\left(\\frac{y}{x}\\right)\\]\nThis is in radian units and we can convert to degrees using\n\\[a = \\frac{180}{\\pi}\\tan^{-1}\\left(\\frac{y}{x}\\right)\\]\nFor the angle to the goal, we need to alter \\(x\\) and \\(y\\) again. We’ll use \\(x^*\\) from above and also use the absolute value of \\(y\\) so that the degrees range from 0 to 180." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#reduce-correlation", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#reduce-correlation", + "title": "2 - Feature Engineering", + "section": "Reduce correlation ", + "text": "Reduce correlation \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#update-parameter-ranges", - "href": "archive/2022-07-RStudio-conf/annotations.html#update-parameter-ranges", - "title": "Annotations", - "section": "Update parameter ranges", - "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nIn our case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_ns(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n\n#> # A tibble: 1 × 5\n#> name call_info source component component_id\n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_ns ns_P1Tjg\n\nns_tunable$call_info\n\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 1 15" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps", + "title": "2 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#spline-grid-search", - "href": "archive/2022-07-RStudio-conf/annotations.html#spline-grid-search", - "title": "Annotations", - "section": "Spline grid search", - "text": "Spline grid search\nWhat’s going on with the\n\nprediction from a rank-deficient fit may be misleading\n\nwarnings?\nFor linear regression, a computation is used called matrix inversion. The matrix in question is called the “model matrix” and it contains the predictor set for the training data.\nMatrix inversion can fail if two or more columns:\n\nare identical, or\nadd up to some other column.\n\nThese situations are called linear dependencies.\nWhen this happens, lm() is pretty tolerant. It does not fail but does not compute regression coefficients for a minimal number of predictors involved in the dependency (and issues the warning above).\nFor these data, there are three dependencies between:\n\ndefense_team_PIT and offense_team_PIT\nstrength_short_handed, player_diff, and strength_power_play\nyear, month_Oct, month_Nov, and month_Dec\n\nThe first one is easy to explain. For each row, when one these two PIT column has a one, the other must have a zero. The linear regression intercept is represented in the model matrix as a column of all ones. The dependency is\n(Intercept) = defense_team_PIT + offense_team_PIT\nThe way to avoid this problem is to use step_lincomb(all_numeric_predictors()) in the recipe. This step removes the minimum number of columns to avoid the issue.\ntl;dr\nLinear regression detects some redundancies in the predictor set. We can ignore the warnings since lm() can deal with it or use step_lincomb() to avoid the warnings." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps-1", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps-1", + "title": "2 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n embed::step_umap(all_numeric_predictors(), outcome = vars(avg_price_per_room))\n\n\nA fancy machine learning supervised dimension reduction technique called UMAP…\n\nNote that this uses the outcome, and it is from an extension package" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#boosted-tree-tuning-parameters", - "href": "archive/2022-07-RStudio-conf/annotations.html#boosted-tree-tuning-parameters", - "title": "Annotations", - "section": "Boosted tree tuning parameters", - "text": "Boosted tree tuning parameters\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps-2", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps-2", + "title": "2 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#boosted-tree-code", - "href": "archive/2022-07-RStudio-conf/annotations.html#boosted-tree-code", - "title": "Annotations", - "section": "Boosted tree code", - "text": "Boosted tree code\nWe set an engine argument called validation here. That’s not an argument to any function in the xgboost package.\nparsnip has its own wrapper around (xgboost::xgb.train()) called xgb_train(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the xgboost entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for xgboost)." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn-1", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn-1", + "title": "2 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\n\nCreate a recipe() for the hotel data to:\n\nuse a Yeo-Johnson (YJ) transformation on lead_time\nconvert factors to indicator variables\nremove zero-variance variables\nadd the spline technique shown above\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#the-final-fit-to-the-nhl-data", - "href": "archive/2022-07-RStudio-conf/annotations.html#the-final-fit-to-the-nhl-data", - "title": "Annotations", - "section": "The final fit to the NHL data", - "text": "The final fit to the NHL data\nRecall that last_fit() uses the objects produced by initial_split() to determine what data are used for the final model fit and which are used as the test set.\nFor the validation set, last_fit() will use the non-testing data to create the final model fit. This includes the training and validation set.\nThere is no agreement in the community on whether this is the best approach or if we should just use the training set. There are good arguments either way.\nIf you only want to use the training set for the final model, you can do this via:\n\ntraining_data <- nhl_val$splits[[1]] %>% analysis()\n\n# Use `fit()` to train the model on just the training set\nfinal_glm_spline_wflow <- \n glm_spline_wflow %>% \n fit(data = training_data)\n\n# Create test set predictions\ntest_set_pred <- augment(final_glm_spline_wflow, nhl_test)\n\n# Setup and compute the test set metrics\ncls_metrics <- metric_set(roc_auc, accuracy)\n\ntest_res <- \n test_set_pred %>% \n cls_metrics(on_goal, estimate = .pred_class, .pred_yes)\ntest_res" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#minimal-recipe", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#minimal-recipe", + "title": "2 - Feature Engineering", + "section": "Minimal recipe ", + "text": "Minimal recipe \n\nhotel_indicators <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_YeoJohnson(lead_time) |>\n step_dummy(all_nominal_predictors()) |>\n step_zv(all_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#explain-yourself", - "href": "archive/2022-07-RStudio-conf/annotations.html#explain-yourself", - "title": "Annotations", - "section": "Explain yourself", - "text": "Explain yourself\nSome other resources:\n\nTMwR chapter Explaining Models and Predictions\nExplanatory Model Analysis book\nInterpretable Machine Learning book\nDefinitions, methods, and applications in interpretable machine learning" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#measuring-performance", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#measuring-performance", + "title": "2 - Feature Engineering", + "section": "Measuring Performance ", + "text": "Measuring Performance \nWe’ll compute two measures: mean absolute error and the coefficient of determination (a.k.a \\(R^2\\)).\n\\[\\begin{align}\nMAE &= \\frac{1}{n}\\sum_{i=1}^n |y_i - \\hat{y}_i| \\notag \\\\\nR^2 &= cor(y_i, \\hat{y}_i)^2\n\\end{align}\\]\nThe focus will be on MAE for parameter optimization. We’ll use a metric set to compute these:\n\nreg_metrics <- metric_set(mae, rsq)" }, { - "objectID": "archive/2022-07-RStudio-conf/annotations.html#a-tidymodels-explainer", - "href": "archive/2022-07-RStudio-conf/annotations.html#a-tidymodels-explainer", - "title": "Annotations", - "section": "A tidymodels explainer", - "text": "A tidymodels explainer\nFor our example, the angle was an original predictor. Recall that we made spline terms from this predictor, so there are derived features such as angle_ns_1 and so on.\nOriginal versus derived doesn’t affect local explainers since we are focused on a single prediction.\nFor global explainers, we should decide between:\n\nexplaining the overall affect of angle (lumping all its features into one importance score), or\nexplaining the effect of each term in the model (including angle_ns_1 and so on).\n\nThe choice depends on what you want. For example, if we have an original date predictor and make features for month and year, is it more informative to know if date is important (overall) or exactly how the date is important? You might want to look at it both ways." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#using-a-workflow", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#using-a-workflow", + "title": "2 - Feature Engineering", + "section": "Using a workflow ", + "text": "Using a workflow \n\nset.seed(9)\n\nhotel_lm_wflow <-\n workflow() |>\n add_recipe(hotel_indicators) |>\n add_model(linear_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nhotel_lm_res <-\n hotel_lm_wflow |>\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_lm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.6 10 0.214 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00339 Preprocessor1_Model1" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#who-are-you", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn-2", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn-2", + "title": "2 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#who-are-tidymodels", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#who-are-tidymodels", - "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#holdout-predictions", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#holdout-predictions", + "title": "2 - Feature Engineering", + "section": "Holdout predictions ", + "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nlm_cv_pred <- collect_predictions(hotel_lm_res)\nlm_cv_pred |> print(n = 7)\n#> # A tibble: 3,749 × 5\n#> .pred id .row avg_price_per_room .config \n#> <dbl> <chr> <int> <dbl> <chr> \n#> 1 75.1 Fold01 20 40 Preprocessor1_Model1\n#> 2 49.3 Fold01 28 54 Preprocessor1_Model1\n#> 3 64.9 Fold01 45 50 Preprocessor1_Model1\n#> 4 52.8 Fold01 49 42 Preprocessor1_Model1\n#> 5 48.6 Fold01 61 49 Preprocessor1_Model1\n#> 6 29.8 Fold01 66 40 Preprocessor1_Model1\n#> 7 36.9 Fold01 88 49 Preprocessor1_Model1\n#> # ℹ 3,742 more rows" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#asking-for-help", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#asking-for-help", - "title": "1 - Introduction", - "section": "Asking for help", - "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#calibration-plot", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#calibration-plot", + "title": "2 - Feature Engineering", + "section": "Calibration Plot ", + "text": "Calibration Plot \n\nlibrary(probably)\n\ncal_plot_regression(hotel_lm_res)" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#section", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#section", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", + "title": "2 - Feature Engineering", + "section": "What do we do with the agent and company data?", + "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 unique companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor (in the extra materials)" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#tentative-plan-for-this-workshop", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#tentative-plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Tentative plan for this workshop", - "text": "Tentative plan for this workshop\n\nToday:\n\nYour data budget\nWhat makes a model\nEvaluating models\n\nTomorrow:\n\nFeature engineering\nTuning hyperparameters\nTransportation case study\nWrapping up!" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#per-agent-statistics", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#per-agent-statistics", + "title": "2 - Feature Engineering", + "section": "Per-agent statistics", + "text": "Per-agent statistics" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#section-1", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#section-1", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋\n\n Log in to RStudio Cloud here (free):\nbit.ly/tidymodels-iceland-2022" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#collapsing-factor-levels", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#collapsing-factor-levels", + "title": "2 - Feature Engineering", + "section": "Collapsing factor levels ", + "text": "Collapsing factor levels \nThere is a recipe step that will redefine factor levels based on their frequency in the training set:\n\nhotel_other_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_YeoJohnson(lead_time) |>\n step_other(agent, threshold = 0.001) |>\n step_dummy(all_nominal_predictors()) |>\n step_zv(all_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nUsing this code, 34 agents (out of 98) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#does-othering-help", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#does-othering-help", + "title": "2 - Feature Engineering", + "section": "Does othering help? ", + "text": "Does othering help? \n\nhotel_other_wflow <-\n hotel_lm_wflow |>\n update_recipe(hotel_other_rec)\n\nhotel_other_res <-\n hotel_other_wflow |>\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.213 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00341 Preprocessor1_Model1\n\nAbout the same MAE and much faster to complete.\nNow let’s look at a more sophisticated tool called effect feature hashing." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning-1", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning-1", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing", + "title": "2 - Feature Engineering", + "section": "Feature Hashing", + "text": "Feature Hashing\nBetween agent and company, simple dummy variables would create 198 new columns (that are mostly zeros).\nAnother option is to have a binary indicator that combines some levels of these variables.\nFeature hashing (for more see FES, SMLTAR, TMwR, and aml4td):\n\nuses the character values of the levels\nconverts them to integer hash values\nuses the integers to assign them to a specific indicator column." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning-2", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-machine-learning-2", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-1", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-1", + "title": "2 - Feature Engineering", + "section": "Feature Hashing", + "text": "Feature Hashing\nSuppose we want to use 32 indicator variables for agent.\nFor a agent with value “Max_Kuhn”, a hashing function converts it to an integer (say 210397726).\nTo assign it to one of the 32 columns, we would use modular arithmetic to assign it to a column:\n\n# For \"Max_Kuhn\" put a '1' in column: \n210397726 %% 32\n#> [1] 30\n\nHash functions are meant to emulate randomness." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#your-turn", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#your-turn", - "title": "1 - Introduction", - "section": "Your turn", - "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-pros", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-pros", + "title": "2 - Feature Engineering", + "section": "Feature Hashing Pros", + "text": "Feature Hashing Pros\n\nThe procedure will automatically work on new values of the predictors.\nIt is fast.\n“Signed” hashes add a sign to help avoid aliasing." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-tidymodels", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-is-tidymodels", - "title": "1 - Introduction", - "section": "What is tidymodels? ", - "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.1.0 ──\n#> ✔ broom 1.0.5 ✔ rsample 1.1.1\n#> ✔ dials 1.2.0 ✔ tibble 3.2.1\n#> ✔ dplyr 1.1.2 ✔ tidyr 1.3.0\n#> ✔ infer 1.0.4 ✔ tune 1.1.1\n#> ✔ modeldata 1.1.0 ✔ workflows 1.1.3\n#> ✔ parsnip 1.1.0 ✔ workflowsets 1.0.1\n#> ✔ purrr 1.0.1 ✔ yardstick 1.2.0\n#> ✔ recipes 1.0.6\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Learn how to get started at https://www.tidymodels.org/start/" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-cons", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-cons", + "title": "2 - Feature Engineering", + "section": "Feature Hashing Cons", + "text": "Feature Hashing Cons\n\nThere is no real logic behind which factor levels are combined.\nWe don’t know how many columns to add (more in the next section).\nSome columns may have all zeros.\nIf a indicator column is important to the model, we can’t easily determine why.\n\n\nThe signed hash make it slightly more possible to differentiate between confounded levels" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#the-whole-game", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#the-whole-game", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\nTomorrow we will walk through a case study in detail to illustrate feature engineering and model tuning.\nToday we will walk through the analysis at a higher level to show the model development process as a whole and give you an introduction to the data set.\nThe data are from the NHL where we want to predict whether a shot was on-goal or not! 🏒\nIt’s a good example to show how model development works." + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes", + "title": "2 - Feature Engineering", + "section": "Feature Hashing in recipes ", + "text": "Feature Hashing in recipes \nThe textrecipes package has a step that can be added to the recipe:\n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) |>\n step_dummy_hash(company) |>\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nhotel_hash_wflow <-\n hotel_lm_wflow |>\n update_recipe(hash_rec)" + }, + { + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", + "title": "2 - Feature Engineering", + "section": "Feature Hashing in recipes ", + "text": "Feature Hashing in recipes \n\nhotel_hash_res <-\n hotel_hash_wflow |>\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_hash_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.239 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00324 Preprocessor1_Model1\n\nAbout the same performance but now we can handle new values." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#shots-on-goal", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#shots-on-goal", - "title": "1 - Introduction", - "section": "Shots on goal", - "text": "Shots on goal" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#debugging-a-recipe", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#debugging-a-recipe", + "title": "2 - Feature Engineering", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. hash_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#data-spending", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#data-spending", - "title": "1 - Introduction", - "section": "Data spending", - "text": "Data spending" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#example", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#example", + "title": "2 - Feature Engineering", + "section": "Example ", + "text": "Example \n\nhash_rec_fit <- prep(hash_rec)\n\n# Get the transformation coefficient\ntidy(hash_rec_fit, number = 1)\n\n# Get the processed data\nbake(hash_rec_fit, hotel_train |> slice(1:3), contains(\"_agent_\"))" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#a-first-model", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#a-first-model", - "title": "1 - Introduction", - "section": "A first model", - "text": "A first model" + "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#more-on-recipes", + "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#more-on-recipes", + "title": "2 - Feature Engineering", + "section": "More on recipes", + "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#starting-point-logistic-regression", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#starting-point-logistic-regression", - "title": "1 - Introduction", - "section": "Starting point: logistic regression", - "text": "Starting point: logistic regression\n\nWe’ll start by using basic logistic regression to predict our binary outcome.\nOur first model will have 12 simple predictor columns.\nOne initial question: there are 632 players taking shots.\nFor logistic regression, do we convert these to binary indicators (a.k.a. “dummies”)?" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---setup", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---setup", + "title": "4 - Grid Search via Racing", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#basic-features-inc-dummy-variables", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#basic-features-inc-dummy-variables", - "title": "1 - Introduction", - "section": "Basic features (inc dummy variables)", - "text": "Basic features (inc dummy variables)" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---data-usage", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---data-usage", + "title": "4 - Grid Search via Racing", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#different-player-encoding", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#different-player-encoding", - "title": "1 - Introduction", - "section": "Different player encoding", - "text": "Different player encoding" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---boosting-model", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---boosting-model", + "title": "4 - Grid Search via Racing", + "section": "Previously - Boosting Model ", + "text": "Previously - Boosting Model \n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) |>\n step_dummy_hash(company, num_terms = tune(\"company hash\")) |>\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow |>\n extract_parameter_set_dials() |>\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#what-about-location", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#what-about-location", - "title": "1 - Introduction", - "section": "What about location", - "text": "What about location\nThe previous models used the x/y coordinates.\nAre there better ways to represent shot location?\nHow can we make location more usable for the model?" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#first-a-shameless-promotion", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#first-a-shameless-promotion", + "title": "4 - Grid Search via Racing", + "section": "First, a shameless promotion", + "text": "First, a shameless promotion" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-angle", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-angle", - "title": "1 - Introduction", - "section": "Add shot angle?", - "text": "Add shot angle?" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#making-grid-search-more-efficient", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#making-grid-search-more-efficient", + "title": "4 - Grid Search via Racing", + "section": "Making Grid Search More Efficient", + "text": "Making Grid Search More Efficient\nIn the last section, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\nAlso, for some models, we can fit far fewer models than the number that are being evaluated.\n\nFor boosting, a model with X trees can often predict on candidates with less than X trees.\n\nBoth of these methods can lead to enormous speed-ups." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-from-defensive-zone", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-from-defensive-zone", - "title": "1 - Introduction", - "section": "Add shot from defensive zone?", - "text": "Add shot from defensive zone?" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#model-racing", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#model-racing", + "title": "4 - Grid Search via Racing", + "section": "Model Racing", + "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models but only for a few resamples.\nDetermine which candidates have a low probability of being selected.\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain)\n\nThis can result in fitting a small number of models." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-behind-goal-line", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#add-shot-behind-goal-line", - "title": "1 - Introduction", - "section": "Add shot behind goal line?", - "text": "Add shot behind goal line?" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#discarding-candidates", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#discarding-candidates", + "title": "4 - Grid Search via Racing", + "section": "Discarding Candidates", + "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large difference between resamples in the results." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#nonlinear-terms-for-angle-and-distance", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#nonlinear-terms-for-angle-and-distance", - "title": "1 - Introduction", - "section": "Nonlinear terms for angle and distance", - "text": "Nonlinear terms for angle and distance" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#resampling-results-non-racing", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#resampling-results-non-racing", + "title": "4 - Grid Search via Racing", + "section": "Resampling Results (Non-Racing)", + "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#try-another-model", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#try-another-model", - "title": "1 - Introduction", - "section": "Try another model", - "text": "Try another model" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#are-candidates-different", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#are-candidates-different", + "title": "4 - Grid Search via Racing", + "section": "Are Candidates Different?", + "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval for the difference in RMSE is (0.99, 2.8), indicating that candidate number 2 has smaller error." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#switch-to-boosting-and-basic-features", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#switch-to-boosting-and-basic-features", - "title": "1 - Introduction", - "section": "Switch to boosting and basic features", - "text": "Switch to boosting and basic features" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#evaluating-differences-in-candidates", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#evaluating-differences-in-candidates", + "title": "4 - Grid Search via Racing", + "section": "Evaluating Differences in Candidates", + "text": "Evaluating Differences in Candidates\n\n\nWhat if we were to have compared the candidates while we seqeuntially evaluated each resample?\n👉\n\nOne candidate shows superiority when 4 resamples have been evaluated." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#boosting-with-location-features", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#boosting-with-location-features", - "title": "1 - Introduction", - "section": "Boosting with location features", - "text": "Boosting with location features" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#interim-analysis-of-results", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#interim-analysis-of-results", + "title": "4 - Grid Search via Racing", + "section": "Interim Analysis of Results", + "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order (so set the seed).\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#choose-wisely", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#choose-wisely", - "title": "1 - Introduction", - "section": "Choose wisely…", - "text": "Choose wisely…" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing", + "title": "4 - Grid Search via Racing", + "section": "Racing ", + "text": "Racing \n\n# Let's use a larger grid\nlgbm_grid <- \n lgbm_param |> \n grid_space_filling(size = 50)\n\nlibrary(finetune)\n\nset.seed(9)\nlgbm_race_res <-\n lgbm_wflow |>\n tune_race_anova(\n resamples = hotel_rs,\n grid = lgbm_grid, \n metrics = reg_metrics\n )\n\nThe syntax and helper functions are extremely similar to those shown for tune_grid()." }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#finalize-and-verify", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#finalize-and-verify", - "title": "1 - Introduction", - "section": "Finalize and verify", - "text": "Finalize and verify" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing-results", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing-results", + "title": "4 - Grid Search via Racing", + "section": "Racing Results ", + "text": "Racing Results \n\nshow_best(lgbm_race_res, metric = \"mae\")\n#> # A tibble: 2 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1347 5 0.0655 66 26 mae standard 9.64 10 0.173 Preprocessor34_Model1\n#> 2 980 8 0.0429 17 135 mae standard 9.76 10 0.164 Preprocessor25_Model1" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#and-so-on", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#and-so-on", - "title": "1 - Introduction", - "section": "… and so on", - "text": "… and so on\nOnce we find an acceptable model and feature set, the process is to\n\nConfirm our results on the test set.\nDocument the data and model development process.\nDeploy, monitor, etc." + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing-results-1", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing-results-1", + "title": "4 - Grid Search via Racing", + "section": "Racing Results ", + "text": "Racing Results \n\n\nOnly 171 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_race_res) + \n scale_x_continuous(breaks = pretty_breaks())" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#lets-install-some-packages", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of RStudio Cloud:\n\ninstall.packages(c(\"Cubist\", \"DALEXtra\", \"doParallel\", \"earth\", \"embed\", \n \"forcats\", \"lme4\", \"parallelly\", \"ranger\", \"remotes\", \"rpart\", \n \"rpart.plot\", \"rules\", \"stacks\", \"tidymodels\",\n \"vetiver\", \"xgboost\"))\n\nremotes::install_github(\"topepo/ongoal@hockeyR\")\n\n\n\n Or log in to RStudio Cloud:\nbit.ly/tidymodels-iceland-2022" + "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#your-turn", + "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#your-turn", + "title": "4 - Grid Search via Racing", + "section": "Your turn", + "text": "Your turn\n\nRun tune_race_anova() with a different seed.\nDid you get the same or similar results?\n\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/01-introduction.html#our-versions", - "href": "archive/2022-08-Reykjavik-City/01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nbroom (1.0.5, CRAN), DALEX (2.4.3, CRAN), DALEXtra (2.2.1, CRAN), dials (1.2.0, CRAN), doParallel (1.0.17, CRAN), dplyr (1.1.2, CRAN), embed (1.1.1, CRAN), ggplot2 (3.4.2, CRAN (R 4.2.1)), modeldata (1.1.0, CRAN), ongoal (0.0.4, Github (topepo/ongoal@2532b2404), parsnip (1.1.0, CRAN), purrr (1.0.1, CRAN), ranger (0.15.1, CRAN), recipes (1.0.6, CRAN), rpart (4.1.16, CRAN (R 4.2.1)), rpart.plot (3.1.1, CRAN), rsample (1.1.1, CRAN), scales (1.2.1, CRAN), stacks (1.0.2, CRAN), tibble (3.2.1, CRAN (R 4.2.1)), tidymodels (1.1.0, CRAN), tidyr (1.3.0, CRAN), tune (1.1.1, CRAN), vetiver (0.2.1, CRAN), workflows (1.1.3, CRAN), workflowsets (1.0.1, CRAN), xgboost (1.7.5.1, CRAN), and yardstick (1.2.0, CRAN)\nQuarto: 1.3.433" + "objectID": "archive/2025-08-nyr/slides/advanced-06-wrapping-up.html#your-turn", + "href": "archive/2025-08-nyr/slides/advanced-06-wrapping-up.html#your-turn", + "title": "6 - Wrapping up", + "section": "Your turn", + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n03:00\n\n\n\n\n\nlm for linear model\nglm for generalized linear model (e.g. logistic regression)\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets" + "objectID": "archive/2025-08-nyr/slides/advanced-06-wrapping-up.html#resources-to-keep-learning", + "href": "archive/2025-08-nyr/slides/advanced-06-wrapping-up.html#resources-to-keep-learning", + "title": "6 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://tidymodels.aml4td.org/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#previously---setup", + "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#previously---setup", + "title": "Extras - Effect Encodings", + "section": "Previously - Setup", + "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-1", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-1", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlinear_reg()\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: lm\n\n\nModels have default engines" + "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#previously---data-usage", + "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#previously---data-usage", + "title": "Extras - Effect Encodings", + "section": "Previously - Data Usage", + "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-2", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-2", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", + "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", + "title": "Extras - Effect Encodings", + "section": "What do we do with the agent and company data?", + "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-3", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-3", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlinear_reg() %>%\n set_engine(\"glmnet\")\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: glmnet" + "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#per-agent-statistics", + "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#per-agent-statistics", + "title": "Extras - Effect Encodings", + "section": "Per-agent statistics", + "text": "Per-agent statistics" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-4", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-4", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlinear_reg() %>%\n set_engine(\"stan\")\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: stan" + "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#what-is-an-effect-encoding", + "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#what-is-an-effect-encoding", + "title": "Extras - Effect Encodings", + "section": "What is an effect encoding?", + "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <fct> <int>\n#> 1 52.7 cynthia_worsley 1\n#> 2 51.8 carlos_bryant 2\n#> 3 53.8 lance_hitchcock 3\n#> 4 51.8 lance_hitchcock 4\n#> 5 46.8 cynthia_worsley 5\n#> 6 54.7 charles_najera 6\n#> 7 46.8 cynthia_worsley 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <dbl> <int>\n#> 1 52.7 88.5 1\n#> 2 51.8 89.5 2\n#> 3 53.8 79.8 3\n#> 4 51.8 79.8 4\n#> 5 46.8 88.5 5\n#> 6 54.7 109. 6\n#> 7 46.8 88.5 7\n\n\nThe agent column is replaced with an estimate of the ADR." + }, + { + "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#per-agent-statistics-again", + "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#per-agent-statistics-again", + "title": "Extras - Effect Encodings", + "section": "Per-agent statistics again", + "text": "Per-agent statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these means use partial pooling.\nPooling borrows strength across agents and shrinks extreme values towards the mean for agents with very few transations\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for agents with fewer reservations by shrinking the estimate to the overall ADR mean" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-5", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-5", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#partial-pooling", + "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#partial-pooling", + "title": "Extras - Effect Encodings", + "section": "Partial pooling", + "text": "Partial pooling" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-6", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-6", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" + "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#agent-effects", + "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#agent-effects", + "title": "Extras - Effect Encodings", + "section": "Agent effects ", + "text": "Agent effects \n\nlibrary(embed)\n\nhotel_effect_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_YeoJohnson(lead_time) |>\n step_lencode_mixed(agent, company, outcome = vars(avg_price_per_room)) |>\n step_dummy(all_nominal_predictors()) |>\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-7", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-7", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"regression\")\n#> Decision Tree Model Specification (regression)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" + "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#effect-encoding-results", + "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#effect-encoding-results", + "title": "Extras - Effect Encodings", + "section": "Effect encoding results ", + "text": "Effect encoding results \n\nhotel_effect_wflow <-\n workflow() |>\n add_model(linear_reg()) |> \n update_recipe(hotel_effect_rec)\n\nreg_metrics <- metric_set(mae, rsq)\n\nhotel_effect_res <-\n hotel_effect_wflow |>\n fit_resamples(hotel_rs, metrics = reg_metrics)\n\ncollect_metrics(hotel_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.8 10 0.189 Preprocessor1_Model1\n#> 2 rsq standard 0.870 10 0.00357 Preprocessor1_Model1\n\nSlightly worse but it can handle new agents (if they occur)." }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-8", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#to-specify-a-model-8", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2025-08-nyr/slides/extras-vetiver.html#deploying-a-model", + "href": "archive/2025-08-nyr/slides/extras-vetiver.html#deploying-a-model", + "title": "Extras - Model deployment", + "section": "Deploying a model ", + "text": "Deploying a model \nWe have a decision tree, tree_fit, to model whether or not a plot of land in Washington is forested or not.\nHow do we use our model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"forested\")\nv\n#> \n#> ── forested ─ <bundled_workflow> model for deployment \n#> A rpart classification modeling workflow using 19 features\n\nLearn more at https://vetiver.rstudio.com" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-1", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-1", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code so it creates a different model.\n\n\n\n05:00\n\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" + "objectID": "archive/2025-08-nyr/slides/extras-vetiver.html#deploy-your-model-1", + "href": "archive/2025-08-nyr/slides/extras-vetiver.html#deploy-your-model-1", + "title": "Extras - Model deployment", + "section": "Deploy your model ", + "text": "Deploy your model \nHow do we use our model in production?\n\nlibrary(plumber)\npr() |>\n vetiver_api(v)\n#> # Plumber router with 4 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Users/emilhvitfeldt/Library/R/arm64/4.5/library/vetiver\n#> ├──/metadata (GET)\n#> ├──/ping (GET)\n#> ├──/predict (POST)\n#> └──/prototype (GET)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#models-well-be-using-today", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#models-well-be-using-today", - "title": "3 - What makes a model?", - "section": "Models we’ll be using today", - "text": "Models we’ll be using today\n\nLinear regression\nDecision trees" + "objectID": "archive/2025-08-nyr/slides/extras-vetiver.html#your-turn", + "href": "archive/2025-08-nyr/slides/extras-vetiver.html#your-turn", + "title": "Extras - Model deployment", + "section": "Your turn", + "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression", - "title": "3 - What makes a model?", - "section": "Linear regression", - "text": "Linear regression" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia", + "title": "2 - Your data budget", + "section": "Data on forests in Georgia", + "text": "Data on forests in Georgia\n\n\n\nThe U.S. Forest Service maintains ML models to predict whether a plot of land is “forested.”\nThis classification is important for all sorts of research, legislation, and land management purposes.\nPlots are typically remeasured every 10 years and this dataset contains the most recent measurement per plot.\nType ?forested_ga to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/251793/forest-mountain" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression-1", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression-1", - "title": "3 - What makes a model?", - "section": "Linear regression", - "text": "Linear regression" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia-1", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia-1", + "title": "2 - Your data budget", + "section": "Data on forests in Georgia", + "text": "Data on forests in Georgia\n\n\n\nN = 10937 plots of land, one from each of 10937 6000-acre hexagons in Georgia.\nA nominal outcome, forested, with levels \"Yes\" and \"No\", measured “on-the-ground.”\n18 remotely-sensed and easily-accessible predictors:\n\nnumeric variables based on weather and topography.\nnominal variables based on classifications from other governmental orgs.\n\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/67614/forest\n\n\n\nThose nominal variables are classifications similar to “forested” but from other agencies. e.g. land_type is from the European Space Agency, and is a remotely-sensed 3-class distribution based on predictions for how the land is used. (The data also includes a county variable, which is not itself from a model.)" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression-2", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#linear-regression-2", - "title": "3 - What makes a model?", - "section": "Linear regression", - "text": "Linear regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nOutcome modeled as linear combination of predictors:\n\n\\(\\mbox{latency} = \\beta_0 + \\beta_1\\cdot\\mbox{age} + \\epsilon\\)\n\nFind a line that minimizes the mean squared error (MSE)" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#checklist-for-predictors", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#checklist-for-predictors", + "title": "2 - Your data budget", + "section": "Checklist for predictors", + "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?\n\n\n\nre: ethics – what issues might arise from releasing the true lat and lon? In reality, these lat and lon are slightly jittered to help ensure trust with landowners who allow surveyers to come take measurements." }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia-2", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia-2", + "title": "2 - Your data budget", + "section": "Data on forests in Georgia", + "text": "Data on forests in Georgia\n\nlibrary(tidymodels)\nlibrary(forested)\n\nforested_ga\n#> # A tibble: 10,937 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 2007 14 0 0 No tree 13.9\n#> 2 Yes 2007 66 -53 10 Tree 13.8\n#> 3 Yes 2006 59 -82 6 No tree 13.5\n#> 4 Yes 2007 116 -78 20 Tree 12.3\n#> 5 Yes 2006 283 63 13 Tree 10.0\n#> 6 Yes 2007 250 63 14 Tree 10.8\n#> 7 Yes 2007 58 31 1 No tree 13.8\n#> 8 Yes 2023 140 56 11 Tree 12.2\n#> 9 Yes 2024 118 72 17 Tree 12.2\n#> 10 Yes 2024 217 -46 13 Tree 12.4\n#> # ℹ 10,927 more rows\n#> # ℹ 12 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>, county <fct>" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees-1", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees-1", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees-2", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#decision-trees-2", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-1", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-1", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "title": "3 - What makes a model?", - "section": "All models are wrong, but some are useful!", - "text": "All models are wrong, but some are useful!\n\n\nLinear regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-2", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-2", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "title": "3 - What makes a model?", - "section": "Workflows bind preprocessors and models", - "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#what-is-wrong-with-this", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#what-is-wrong-with-this", - "title": "3 - What makes a model?", - "section": "What is wrong with this?", - "text": "What is wrong with this?" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-initial-split", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-initial-split", + "title": "2 - Your data budget", + "section": "The initial split ", + "text": "The initial split \n\nset.seed(123)\nforested_split <- initial_split(forested_ga)\nforested_split\n#> <Training/Testing/Total>\n#> <8202/2735/10937>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#why-a-workflow", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#why-a-workflow", - "title": "3 - What makes a model?", - "section": "Why a workflow()? ", - "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering tomorrow!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#what-is-set.seed", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#what-is-set.seed", + "title": "2 - Your data budget", + "section": "What is set.seed()?", + "text": "What is set.seed()?\nTo create that split of the data, R generates “pseudo-random” numbers: while they are made to behave like random numbers, their generation is deterministic given a “seed”.\nThis allows us to reproduce results by setting that seed.\nWhich seed you pick doesn’t matter, as long as you don’t try a bunch of seeds and pick the one that gives you the best performance." }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-1", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-1", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\ntree_spec %>% \n fit(latency ~ ., data = frog_train) \n#> parsnip model object\n#> \n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#accessing-the-data", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#accessing-the-data", + "title": "2 - Your data budget", + "section": "Accessing the data ", + "text": "Accessing the data \n\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-2", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-2", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\nworkflow() %>%\n add_formula(latency ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = frog_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-training-set", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-training-set", + "title": "2 - Your data budget", + "section": "The training set", + "text": "The training set\n\nforested_train\n#> # A tibble: 8,202 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 1997 66 82 10 Tree 12.2 \n#> 2 No 1997 284 -99 58 Tree 10.3 \n#> 3 Yes 2022 130 86 15 Tree 11.8 \n#> 4 Yes 2021 202 -55 3 Tree 10.7 \n#> 5 Yes 1995 75 -89 1 Tree 13.8 \n#> 6 No 1995 110 -53 5 Tree 12.4 \n#> 7 Yes 2022 111 73 12 Tree 11.5 \n#> 8 Yes 1997 230 96 14 Tree 9.98\n#> 9 Yes 2002 160 -88 13 Tree 11.1 \n#> 10 Yes 2020 39 9 6 Tree 13.9 \n#> # ℹ 8,192 more rows\n#> # ℹ 12 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>, county <fct>" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-3", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#a-model-workflow-3", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\nworkflow(latency ~ ., tree_spec) %>% \n fit(data = frog_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-test-set", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-test-set", + "title": "2 - Your data budget", + "section": "The test set ", + "text": "The test set \n🙈\n\nThere are 2735 rows and 19 columns in the test set." }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-2", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-2", - "title": "3 - What makes a model?", + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn-1", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn-1", + "title": "2 - Your data budget", "section": "Your turn", - "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code so it uses a linear model.\n\n\n\n05:00" + "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#predict-with-your-model", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#predict-with-your-model", - "title": "3 - What makes a model?", - "section": "Predict with your model ", - "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\ntree_fit <-\n workflow(latency ~ ., tree_spec) %>% \n fit(data = frog_train)" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-3", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-3", + "title": "2 - Your data budget", + "section": "Data splitting and spending ", + "text": "Data splitting and spending \n\nset.seed(123)\nforested_split <- initial_split(forested_ga, prop = 0.8)\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)\n\nnrow(forested_train)\n#> [1] 8749\nnrow(forested_test)\n#> [1] 2188" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-3", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-3", - "title": "3 - What makes a model?", + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn-2", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn-2", + "title": "2 - Your data budget", "section": "Your turn", - "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = frog_test)\nWhat do you get?\n\n\n\n03:00" + "text": "Your turn\n\nExplore the forested_train data on your own!\n\nWhat’s the distribution of the outcome, forested?\nWhat’s the distribution of numeric variables like precip_annual?\nHow does the distribution of forested differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-4", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-4", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = frog_test)\nWhat do you get?\n\n\n\n03:00" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-1", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-1", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = forested)) +\n geom_bar()" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-2", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-2", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = forested, fill = tree_no_tree)) +\n geom_bar()" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model-1", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model-1", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\nroundint = FALSE is only to quiet a warning" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-3", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-3", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"identity\", alpha = .7)" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model-2", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#understand-your-model-2", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-4", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-4", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"fill\")" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-5", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-5", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nExtract the model engine object from your fitted linear workflow.\n⚠️ Never predict() with any extracted components!\n\n\n\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-5", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-5", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train |> \n ggplot(aes(x = lon, y = lat, col = forested)) +\n geom_point()" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#deploying-a-model", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#deploying-a-model", - "title": "3 - What makes a model?", - "section": "Deploying a model ", - "text": "Deploying a model \nHow do you use your new tree_fit model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"frog_hatching\")\nv\n#> \n#> ── frog_hatching ─ <bundled_workflow> model for deployment \n#> A rpart regression modeling workflow using 4 features\n\nLearn more at https://vetiver.rstudio.com" + "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-whole-game---status-update", + "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-whole-game---status-update", + "title": "2 - Your data budget", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#deploy-your-model-1", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#deploy-your-model-1", - "title": "3 - What makes a model?", - "section": "Deploy your model ", - "text": "Deploy your model \nHow do you use your new model tree_fit in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 3 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Users/emilhvitfeldt/Library/R/arm64/4.2/library/vetiver\n#> ├──/metadata (GET)\n#> ├──/ping (GET)\n#> └──/predict (POST)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#looking-at-predictions", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#looking-at-predictions", + "title": "4 - Evaluating models", + "section": "Looking at predictions", + "text": "Looking at predictions\n\naugment(forested_fit, new_data = forested_train)\n#> # A tibble: 8,749 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness roughness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 0.931 0.0690 Yes 1997 66 82 10\n#> 2 Yes 0.983 0.0172 No 1997 284 -99 58\n#> 3 Yes 0.960 0.0401 Yes 2022 130 86 15\n#> 4 Yes 0.870 0.130 Yes 2021 202 -55 3\n#> 5 Yes 0.823 0.177 Yes 1995 75 -89 1\n#> 6 Yes 0.758 0.242 No 1995 110 -53 5\n#> 7 Yes 0.823 0.177 Yes 2022 111 73 12\n#> 8 No 0.467 0.533 Yes 1997 230 96 14\n#> 9 Yes 0.983 0.0172 Yes 2002 160 -88 13\n#> 10 Yes 0.871 0.129 Yes 2020 39 9 6\n#> # ℹ 8,739 more rows\n#> # ℹ 14 more variables: tree_no_tree <fct>, dew_temp <dbl>, precip_annual <dbl>,\n#> # temp_annual_mean <dbl>, temp_annual_min <dbl>, temp_annual_max <dbl>,\n#> # temp_january_min <dbl>, vapor_min <dbl>, vapor_max <dbl>,\n#> # canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>, county <fct>" }, { - "objectID": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-6", - "href": "archive/2022-08-Reykjavik-City/03-what-makes-a-model.html#your-turn-6", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n05:00" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#working-with-our-predictors", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#working-with-our-predictors", - "title": "5 - Feature engineering", - "section": "Working with our predictors", - "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for lm()).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix-1", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) |>\n conf_mat(truth = forested, estimate = .pred_class)\n#> Truth\n#> Prediction Yes No\n#> Yes 5884 869\n#> No 438 1558" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-is-feature-engineering", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-is-feature-engineering", - "title": "5 - Feature engineering", - "section": "What is feature engineering?", - "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPCA feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix-2", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix-2", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) |>\n conf_mat(truth = forested, estimate = .pred_class) |>\n autoplot(type = \"heatmap\")" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#example-dates", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#example-dates", - "title": "5 - Feature engineering", - "section": "Example: Dates", - "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen a date column is used in its native format, it is usually converted by an R model to an integer.\n\n\nIt can be re-engineered as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#general-definitions", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#general-definitions", - "title": "5 - Feature engineering", - "section": "General definitions ", - "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\nIn a little bit, we’ll see successful (and unsuccessful) feature engineering methods for our example data.\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-accuracy", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-accuracy", + "title": "4 - Evaluating models", + "section": "Dangers of accuracy ", + "text": "Dangers of accuracy \nWe need to be careful of using accuracy() since it can give “good” performance by only predicting one way with imbalanced data:\n\naugment(forested_fit, new_data = forested_train) %>%\n mutate(.pred_class = factor(\"Yes\", levels = c(\"Yes\", \"No\"))) %>%\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.723" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#the-nhl-data", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#the-nhl-data", - "title": "5 - Feature engineering", - "section": "The NHL data 🏒", - "text": "The NHL data 🏒\n\nFrom Pittsburgh Penguins games, 8,915 shots\nData from the 2015-2016 season\n\n\nLet’s predict whether a shot is on-goal (a goal or blocked by goaltender) or not." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-1", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.931" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#case-study", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#case-study", - "title": "5 - Feature engineering", - "section": "Case study", - "text": "Case study\n\nlibrary(tidymodels)\nlibrary(ongoal)\n\ntidymodels_prefer()\n\nglimpse(season_2015)\n#> Rows: 8,915\n#> Columns: 13\n#> $ on_goal <fct> no, no, yes, no, yes, no, no, yes, no, no, no, yes, no…\n#> $ period <int> 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, …\n#> $ game_seconds <dbl> 13, 36, 47, 92, 99, 125, 179, 220, 252, 309, 413, 427,…\n#> $ strength <fct> even, even, even, even, even, even, even, even, even, …\n#> $ home_skaters <dbl> 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, …\n#> $ away_skaters <dbl> 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, …\n#> $ goaltender <fct> marc_andre_fleury, antti_niemi, antti_niemi, antti_nie…\n#> $ goal_difference <dbl> 0, 0, 0, 0, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, -1, -1, 1,…\n#> $ shooter <fct> evgeni_malkin, valeri_nichushkin, phil_kessel, beau_be…\n#> $ shooter_type <fct> center, right_wing, center, right_wing, center, defens…\n#> $ coord_x <dbl> -66, -49, 64, 65, 80, 42, -55, 62, -67, -58, 76, 56, -…\n#> $ coord_y <dbl> -11, -21, -31, -21, 13, 31, -19, 15, -9, -16, -8, 25, …\n#> $ extra_attacker <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-2", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-2", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.931\n\n\n\naugment(forested_fit, new_data = forested_train) |>\n specificity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.642" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#data-splitting-strategy", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#data-splitting-strategy", - "title": "5 - Feature engineering", - "section": "Data splitting strategy", - "text": "Data splitting strategy" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-3", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-3", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\nforested_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(forested_fit, new_data = forested_train) |>\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n#> 2 specificity binary 0.642\n#> 3 sensitivity binary 0.931" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#why-a-validation-set", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#why-a-validation-set", - "title": "5 - Feature engineering", - "section": "Why a validation set?", - "text": "Why a validation set?\nRecall that resampling gives us performance measures without using the test set.\nIt’s important to get good resampling statistics (e.g. \\(R^2\\)).\n\nThat usually means having enough data to estimate performance.\n\nWhen you have “a lot” of data, a validation set can be an efficient way to do this." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-4", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-4", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \nMetrics and metric sets work with grouped data frames!\n\naugment(forested_fit, new_data = forested_train) |>\n group_by(tree_no_tree) |>\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 2 × 4\n#> tree_no_tree .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 Tree accuracy binary 0.875\n#> 2 No tree accuracy binary 0.807" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#splitting-the-nhl-data", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#splitting-the-nhl-data", - "title": "5 - Feature engineering", - "section": "Splitting the NHL data ", - "text": "Splitting the NHL data \n\nset.seed(23)\nnhl_split <- initial_split(season_2015, prop = 3/4)\nnhl_split\n#> <Training/Testing/Total>\n#> <6686/2229/8915>\n\nnhl_train_and_val <- training(nhl_split)\nnhl_test <- testing(nhl_split)\n\n## not testing\nnrow(nhl_train_and_val)\n#> [1] 6686\n \n## testing\nnrow(nhl_test)\n#> [1] 2229" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nApply the forested_metrics metric set to augment()\noutput grouped by tree_no_tree.\nDo any metrics differ substantially between groups?\n\n\n\n−+\n05:00\n\n\n\n\nThe specificity for \"Tree\" is a good bit lower than it is for \"No tree\".\nSpecificity is the proportion of negatives that are correctly identified as negatives. “Negative” is the non-event level of the outcome, i.e. “non-forested.” So, when this index classifies the plot as having a tree, the model does not do well at correctly identifying the plot as non-forested when it is indeed non-forested." }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#validation-split", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#validation-split", - "title": "5 - Feature engineering", - "section": "Validation split ", - "text": "Validation split \nSince there are a lot of observations, we’ll use a validation set:\n\nset.seed(234)\nnhl_val <- validation_split(nhl_train_and_val, prop = 0.80)\nnhl_val\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5348/1338]> validation\n\n\nRemember that a validation split is a type of resample." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#two-class-data", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#two-class-data", + "title": "4 - Evaluating models", + "section": "Two class data", + "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn", - "title": "5 - Feature engineering", - "section": "Your turn", - "text": "Your turn\n\nLet’s explore the training set data.\nUse the function plot_nhl_shots() for nice spatial plots of the data.\n\n\n\nnhl_train <- analysis(nhl_val$splits[[1]])\n\nset.seed(100)\nnhl_train %>% \n sample_n(200) %>%\n plot_nhl_shots(emphasis = shooter_type)\n\n\n\n\n\n\n\n\n\n\n\n\n08:00" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#varying-the-threshold", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#varying-the-threshold", + "title": "4 - Evaluating models", + "section": "Varying the threshold", + "text": "Varying the threshold" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#prepare-your-data-for-modeling", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#prepare-your-data-for-modeling", - "title": "5 - Feature engineering", - "section": "Prepare your data for modeling ", - "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of feature engineering steps that provide preprocessing tools to be applied to data.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves", + "title": "4 - Evaluating models", + "section": "ROC curves", + "text": "ROC curves\n\n\nFor an ROC (receiver operator characteristic) curve, we plot\n\nthe false positive rate (1 - specificity) on the x-axis\nthe true positive rate (sensitivity) on the y-axis\n\nwith sensitivity and specificity calculated at all possible thresholds.\n\n\n\n\n\n\n\n\n\n\n\n\nROC curves are insensitive to class imbalance." }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#a-first-recipe", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#a-first-recipe", - "title": "5 - Feature engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves-1", + "title": "4 - Evaluating models", + "section": "ROC curves", + "text": "ROC curves\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#a-first-recipe-1", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#a-first-recipe-1", - "title": "5 - Feature engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nsummary(nhl_rec)\n#> # A tibble: 13 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 period <chr [2]> predictor original\n#> 2 game_seconds <chr [2]> predictor original\n#> 3 strength <chr [3]> predictor original\n#> 4 home_skaters <chr [2]> predictor original\n#> 5 away_skaters <chr [2]> predictor original\n#> 6 goaltender <chr [3]> predictor original\n#> 7 goal_difference <chr [2]> predictor original\n#> 8 shooter <chr [3]> predictor original\n#> 9 shooter_type <chr [3]> predictor original\n#> 10 coord_x <chr [2]> predictor original\n#> 11 coord_y <chr [2]> predictor original\n#> 12 extra_attacker <chr [2]> predictor original\n#> 13 on_goal <chr [3]> outcome original" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves-2", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves-2", + "title": "4 - Evaluating models", + "section": "ROC curves ", + "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(forested_fit, new_data = forested_train) |> \n roc_curve(truth = forested, .pred_Yes) |>\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.143 0.267 0.990\n#> 3 0.385 0.571 0.951\n\naugment(forested_fit, new_data = forested_train) |> \n roc_auc(truth = forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.881" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#create-indicator-variables", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#create-indicator-variables", - "title": "5 - Feature engineering", - "section": "Create indicator variables ", - "text": "Create indicator variables \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curve-plot", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curve-plot", + "title": "4 - Evaluating models", + "section": "ROC curve plot ", + "text": "ROC curve plot \n\n\naugment(forested_fit, \n new_data = forested_train) |> \n roc_curve(truth = forested, \n .pred_Yes) |>\n autoplot()" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#filter-out-constant-columns", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#filter-out-constant-columns", - "title": "5 - Feature engineering", - "section": "Filter out constant columns ", - "text": "Filter out constant columns \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-1", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#normalization", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#normalization", - "title": "5 - Feature engineering", - "section": "Normalization ", - "text": "Normalization \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#brier-score", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#brier-score", + "title": "4 - Evaluating models", + "section": "Brier score", + "text": "Brier score\nWhat if we don’t turn predicted probabilities into class predictions?\n\nThe Brier score is analogous to the mean squared error in regression models:\n\\[\nBrier_{class} = \\frac{1}{N}\\sum_{i=1}^N\\sum_{k=1}^C (y_{ik} - \\hat{p}_{ik})^2\n\\]" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#reduce-correlation", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#reduce-correlation", - "title": "5 - Feature engineering", - "section": "Reduce correlation ", - "text": "Reduce correlation \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#brier-score-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#brier-score-1", + "title": "4 - Evaluating models", + "section": "Brier score", + "text": "Brier score\n\naugment(forested_fit, new_data = forested_train) |> \n brier_class(truth = forested, .pred_Yes) \n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.113\n\n\nSmaller values are better, for binary classification the “bad model threshold” is about 0.25." }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps", - "title": "5 - Feature engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#separation-vs-calibration", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#separation-vs-calibration", + "title": "4 - Evaluating models", + "section": "Separation vs calibration", + "text": "Separation vs calibration\n\n\nThe ROC captures separation.\n\n\n\n\n\n\n\n\n\n\nThe Brier score captures calibration.\n\n\n\n\n\n\n\n\n\n\n\n\nGood separation: the densities don’t overlap.\nGood calibration: the calibration line follows the diagonal.\n\nCalibration plot: We bin observations according to predicted probability. In the bin for 20%-30% predicted prob, we should see an event rate of ~25% if the model is well-calibrated." }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps-1", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps-1", - "title": "5 - Feature engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = on_goal)\n\n\nA fancy machine learning supervised dimension reduction technique…\n\nNote that this uses the outcome, and it is from an extension package" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-1", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting", + "text": "Dangers of overfitting" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps-2", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#other-possible-steps-2", - "title": "5 - Feature engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_ns(coord_y, coord_x, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-2", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-2", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" + }, + { + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-3", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-3", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\nforested_fit |>\n augment(forested_train)\n#> # A tibble: 8,749 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness roughness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 0.931 0.0690 Yes 1997 66 82 10\n#> 2 Yes 0.983 0.0172 No 1997 284 -99 58\n#> 3 Yes 0.960 0.0401 Yes 2022 130 86 15\n#> 4 Yes 0.870 0.130 Yes 2021 202 -55 3\n#> 5 Yes 0.823 0.177 Yes 1995 75 -89 1\n#> 6 Yes 0.758 0.242 No 1995 110 -53 5\n#> 7 Yes 0.823 0.177 Yes 2022 111 73 12\n#> 8 No 0.467 0.533 Yes 1997 230 96 14\n#> 9 Yes 0.983 0.0172 Yes 2002 160 -88 13\n#> 10 Yes 0.871 0.129 Yes 2020 39 9 6\n#> # ℹ 8,739 more rows\n#> # ℹ 14 more variables: tree_no_tree <fct>, dew_temp <dbl>, precip_annual <dbl>,\n#> # temp_annual_mean <dbl>, temp_annual_min <dbl>, temp_annual_max <dbl>,\n#> # temp_january_min <dbl>, vapor_min <dbl>, vapor_max <dbl>,\n#> # canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>, county <fct>\n\nWe call this “resubstitution” or “repredicting the training set”" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-1", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-1", - "title": "5 - Feature engineering", - "section": "Your turn", - "text": "Your turn\n\nCreate a recipe() for the on-goal data to :\n\ncreate one-hot indicator variables\nremove zero-variance variables\n\n\n\n\n03:00" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-4", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-4", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n\nWe call this a “resubstitution estimate”" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#minimal-recipe", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#minimal-recipe", - "title": "5 - Feature engineering", - "section": "Minimal recipe ", - "text": "Minimal recipe \n\nnhl_indicators <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-5", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-5", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#using-a-workflow", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#using-a-workflow", - "title": "5 - Feature engineering", - "section": "Using a workflow ", - "text": "Using a workflow \n\nset.seed(9)\n\nnhl_glm_wflow <-\n workflow() %>%\n add_recipe(nhl_indicators) %>%\n add_model(logistic_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nnhl_glm_res <-\n nhl_glm_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_glm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.756 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.753 1 NA Preprocessor1_Model1" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-6", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-6", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n\n\n\nforested_fit |>\n augment(forested_test) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.711\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-2", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-2", - "title": "5 - Feature engineering", + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-2", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-2", + "title": "4 - Evaluating models", "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n05:00" - }, - { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#holdout-predictions", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#holdout-predictions", - "title": "5 - Feature engineering", - "section": "Holdout predictions ", - "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nglm_val_pred <- collect_predictions(nhl_glm_res)\nglm_val_pred %>% slice(1:7)\n#> # A tibble: 7 × 7\n#> id .pred_yes .pred_no .row .pred_class on_goal .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 validation 0.00844 0.992 2 no no Preprocessor1_Mod…\n#> 2 validation 0.0864 0.914 10 no no Preprocessor1_Mod…\n#> 3 validation 0.799 0.201 22 yes yes Preprocessor1_Mod…\n#> 4 validation 0.615 0.385 24 yes yes Preprocessor1_Mod…\n#> 5 validation 0.787 0.213 31 yes yes Preprocessor1_Mod…\n#> 6 validation 0.872 0.128 39 yes yes Preprocessor1_Mod…\n#> 7 validation 0.00000000903 1.00 40 no no Preprocessor1_Mod…" + "text": "Your turn\n\nUse augment() and a metric function to compute a classification metric like brier_class().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#two-class-data-1", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#two-class-data-1", - "title": "5 - Feature engineering", - "section": "Two class data", - "text": "Two class data\nThese definitions assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-7", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-7", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n brier_class(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.113\n\n\n\nforested_fit |>\n augment(forested_test) |>\n brier_class(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.208\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#varying-the-threshold", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#varying-the-threshold", - "title": "5 - Feature engineering", - "section": "Varying the threshold", - "text": "Varying the threshold" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curves", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curves", - "title": "5 - Feature engineering", - "section": "ROC curves", - "text": "ROC curves\nTo make an ROC (receiver operator characteristic) curve, we:\n\ncalculate the sensitivity and specificity for all possible thresholds\nplot false positive rate (x-axis) versus true positive rate (y-axis)\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-1", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curves-1", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curves-1", - "title": "5 - Feature engineering", - "section": "ROC curves ", - "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\nroc_curve_points <- glm_val_pred %>% roc_curve(truth = on_goal, .pred_yes)\nroc_curve_points %>% slice(1, 50, 100)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.00143 0.0412 0.968\n#> 3 0.00988 0.127 0.968\n\nglm_val_pred %>% roc_auc(truth = on_goal, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.753" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-3", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-3", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curve-plot", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#roc-curve-plot", - "title": "5 - Feature engineering", - "section": "ROC curve plot ", - "text": "ROC curve plot \n\nautoplot(roc_curve_points)" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-2", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-2", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(forested_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7874/875]> Fold01\n#> 2 <split [7874/875]> Fold02\n#> 3 <split [7874/875]> Fold03\n#> 4 <split [7874/875]> Fold04\n#> 5 <split [7874/875]> Fold05\n#> 6 <split [7874/875]> Fold06\n#> 7 <split [7874/875]> Fold07\n#> 8 <split [7874/875]> Fold08\n#> 9 <split [7874/875]> Fold09\n#> 10 <split [7875/874]> Fold10" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-3", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-3", - "title": "5 - Feature engineering", - "section": "Your turn", - "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n05:00" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-3", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-3", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWhat is in this?\n\nforested_folds <- vfold_cv(forested_train)\nforested_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n\n\nTalk about a list column, storing non-atomic types in dataframe" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-do-we-do-with-the-player-data", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-do-we-do-with-the-player-data", - "title": "5 - Feature engineering", - "section": "What do we do with the player data? 🏒", - "text": "What do we do with the player data? 🏒\nThere are 598 unique player values in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump players who rarely shoot into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the shooter column with the estimated effect of that predictor\n\n\n\nLet’s look at othering then effect encodings." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-4", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-4", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(forested_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [6999/1750]> Fold1\n#> 2 <split [6999/1750]> Fold2\n#> 3 <split [6999/1750]> Fold3\n#> 4 <split [6999/1750]> Fold4\n#> 5 <split [7000/1749]> Fold5" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#per-player-statistics", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#per-player-statistics", - "title": "5 - Feature engineering", - "section": "Per-player statistics", - "text": "Per-player statistics" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-5", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-5", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nforested_folds <- vfold_cv(forested_train, v = 10)\nforested_folds\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7874/875]> Fold01\n#> 2 <split [7874/875]> Fold02\n#> 3 <split [7874/875]> Fold03\n#> 4 <split [7874/875]> Fold04\n#> 5 <split [7874/875]> Fold05\n#> 6 <split [7874/875]> Fold06\n#> 7 <split [7874/875]> Fold07\n#> 8 <split [7874/875]> Fold08\n#> 9 <split [7874/875]> Fold09\n#> 10 <split [7875/874]> Fold10\n\n\nSet the seed when creating resamples" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#collapsing-factor-levels", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#collapsing-factor-levels", - "title": "5 - Feature engineering", - "section": "Collapsing factor levels", - "text": "Collapsing factor levels\nThere is a recipe step that will redefine factor levels based on the their frequency in the training set:\n\nnhl_other_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n # Any player with <= 0.01% of shots is set to \"other\"\n step_other(shooter, threshold = 0.001) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\nUsing this code, 402 players (out of 598) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", + "title": "4 - Evaluating models", + "section": "Fit our model to the resamples", + "text": "Fit our model to the resamples\n\nforested_res <- fit_resamples(forested_wflow, forested_folds)\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]>" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#does-othering-help", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#does-othering-help", - "title": "5 - Feature engineering", - "section": "Does othering help?", - "text": "Does othering help?\n\nnhl_other_wflow <-\n nhl_glm_wflow %>%\n update_recipe(nhl_other_rec)\n\nnhl_other_res <-\n nhl_other_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.778 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.804 1 NA Preprocessor1_Model1\n\nA little better ROC AUC and much faster to complete.\nNow let’s look at a more sophisticated tool called effect encodings." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nforested_res |>\n collect_metrics()\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.704 10 0.00653 Preprocessor1_Model1\n#> 2 brier_class binary 0.214 10 0.00349 Preprocessor1_Model1\n#> 3 roc_auc binary 0.692 10 0.00496 Preprocessor1_Model1\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nWe can reliably measure performance using only the training data 🎉" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-is-an-effect-encoding", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#what-is-an-effect-encoding", - "title": "5 - Feature engineering", - "section": "What is an effect encoding?", - "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> on_goal shooter .row\n#> <fct> <fct> <int>\n#> 1 yes sidney_crosby 1\n#> 2 yes mike_hoffman 2\n#> 3 yes ian_cole 3\n#> 4 yes matt_cullen 4\n#> 5 yes kris_letang 5\n#> 6 no nazem_kadri 6\n#> 7 yes david_perron 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> on_goal shooter .row\n#> <fct> <dbl> <int>\n#> 1 yes 0.220 1\n#> 2 yes 0.302 2\n#> 3 yes 0.261 3\n#> 4 yes 0.463 4\n#> 5 yes 0.197 5\n#> 6 no 0.302 6\n#> 7 yes 0.400 7\n\n\nThe shooter column is replaced with the log-odds of being on goal.\n\nAs a reminder:\n\\[\\text{log-odds} = log\\left(\\frac{\\hat{p}}{1 - \\hat{p}}\\right)\\]\nwhere \\(\\hat{p}\\) is the on goal rate estimate.\nFor logistic regression, this is what the predictors are modeling. The log-odds are more likely to be linear with the outcome." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#comparing-metrics", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#comparing-metrics", + "title": "4 - Evaluating models", + "section": "Comparing metrics ", + "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\nforested_res |>\n collect_metrics() |> \n select(.metric, mean, n)\n#> # A tibble: 3 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.704 10\n#> 2 brier_class 0.214 10\n#> 3 roc_auc 0.692 10\n\n\nThe ROC AUC previously was\n\n0.88 for the training set\n0.7 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#per-player-statistics-again", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#per-player-statistics-again", - "title": "5 - Feature engineering", - "section": "Per-player statistics again", - "text": "Per-player statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these rates use partial pooling.\nPooling borrows strength across players and shrinks extreme values (e.g. zero or one) towards the mean for players with very few shots.\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for players with fewer shots by shrinking the estimate to the overall on-goal rate (56.5%)" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-1", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_forested <- control_resamples(save_pred = TRUE)\nforested_res <- fit_resamples(forested_wflow, forested_folds, control = ctrl_forested)\n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]> <tibble>" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#partial-pooling", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#partial-pooling", - "title": "5 - Feature engineering", - "section": "Partial pooling", - "text": "Partial pooling" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-2", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-2", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nforested_preds <- collect_predictions(forested_res)\nforested_preds\n#> # A tibble: 8,749 × 7\n#> .pred_class .pred_Yes .pred_No id .row forested .config \n#> <fct> <dbl> <dbl> <chr> <int> <fct> <chr> \n#> 1 Yes 0.953 0.0473 Fold01 2 No Preprocessor1_Model1\n#> 2 Yes 0.6 0.4 Fold01 6 No Preprocessor1_Model1\n#> 3 Yes 0.848 0.152 Fold01 7 Yes Preprocessor1_Model1\n#> 4 Yes 0.941 0.0588 Fold01 36 Yes Preprocessor1_Model1\n#> 5 Yes 0.895 0.105 Fold01 38 No Preprocessor1_Model1\n#> 6 Yes 1 0 Fold01 58 Yes Preprocessor1_Model1\n#> 7 No 0.187 0.812 Fold01 69 No Preprocessor1_Model1\n#> 8 Yes 0.905 0.0952 Fold01 71 Yes Preprocessor1_Model1\n#> 9 Yes 0.907 0.0930 Fold01 74 Yes Preprocessor1_Model1\n#> 10 Yes 0.904 0.0962 Fold01 80 Yes Preprocessor1_Model1\n#> # ℹ 8,739 more rows" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#player-effects", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#player-effects", - "title": "5 - Feature engineering", - "section": "Player effects ", - "text": "Player effects \n\nlibrary(embed)\n\nnhl_effect_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_lencode_mixed(shooter, goaltender, outcome = vars(on_goal)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-3", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-3", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nforested_preds |> \n group_by(id) |>\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.729\n#> 2 Fold02 accuracy binary 0.685\n#> 3 Fold03 accuracy binary 0.687\n#> 4 Fold04 accuracy binary 0.711\n#> 5 Fold05 accuracy binary 0.736\n#> 6 Fold06 accuracy binary 0.670\n#> 7 Fold07 accuracy binary 0.699\n#> 8 Fold08 accuracy binary 0.710\n#> 9 Fold09 accuracy binary 0.699\n#> 10 Fold10 accuracy binary 0.719\n#> # ℹ 20 more rows" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#recipes-are-estimated", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#recipes-are-estimated", - "title": "5 - Feature engineering", - "section": "Recipes are estimated ", - "text": "Recipes are estimated \nPreprocessing steps in a recipe use the training set to compute quantities.\n\nWhat kind of quantities are computed for preprocessing?\n\nLevels of a factor\nWhether a column has zero variance\nNormalization\nFeature extraction\nEffect encodings\n\n\n\nWhen a recipe is part of a workflow, this estimation occurs when fit() is called." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#where-are-the-fitted-models", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#where-are-the-fitted-models", + "title": "4 - Evaluating models", + "section": "Where are the fitted models? ", + "text": "Where are the fitted models? \n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#effect-encoding-results", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#effect-encoding-results", - "title": "5 - Feature engineering", - "section": "Effect encoding results ", - "text": "Effect encoding results \n\nnhl_effect_wflow <-\n nhl_glm_wflow %>%\n update_recipe(nhl_effect_rec)\n\nnhl_effect_res <-\n nhl_effect_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.791 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.805 1 NA Preprocessor1_Model1\n\nBetter and it can handle new players (if they occur)." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#bootstrapping", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#bootstrapping", + "title": "4 - Evaluating models", + "section": "Bootstrapping", + "text": "Bootstrapping" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#angle", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#angle", - "title": "5 - Feature engineering", - "section": "Angle", - "text": "Angle\n\nnhl_angle_rec <-\n nhl_effect_rec %>%\n step_mutate(\n angle = abs( atan2(abs(coord_y), (89 - coord_x) ) * (180 / pi) )\n )\n\n\n\nNote the danger of using step_mutate() – easy to have data leakage" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#bootstrapping-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#bootstrapping-1", + "title": "4 - Evaluating models", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(forested_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [8749/3218]> Bootstrap01\n#> 2 <split [8749/3264]> Bootstrap02\n#> 3 <split [8749/3220]> Bootstrap03\n#> 4 <split [8749/3208]> Bootstrap04\n#> 5 <split [8749/3230]> Bootstrap05\n#> 6 <split [8749/3197]> Bootstrap06\n#> 7 <split [8749/3193]> Bootstrap07\n#> 8 <split [8749/3226]> Bootstrap08\n#> 9 <split [8749/3243]> Bootstrap09\n#> 10 <split [8749/3233]> Bootstrap10\n#> # ℹ 15 more rows" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#shot-from-the-defensive-zone", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#shot-from-the-defensive-zone", - "title": "5 - Feature engineering", - "section": "Shot from the defensive zone", - "text": "Shot from the defensive zone\n\nnhl_zone_rec <-\n nhl_angle_rec %>%\n step_mutate(\n defensive_zone = ifelse(coord_x <= -25.5, 1, 0)\n )" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game---status-update", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game---status-update", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#behind-goal-line", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#behind-goal-line", - "title": "5 - Feature engineering", - "section": "Behind goal line", - "text": "Behind goal line\n\nnhl_behind_rec <-\n nhl_zone_rec %>%\n step_mutate(\n behind_goal_line = ifelse(coord_x >= 89, 1, 0)\n )" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-4", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-4", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the functions)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#fit-different-recipes", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#fit-different-recipes", - "title": "5 - Feature engineering", - "section": "Fit different recipes ", - "text": "Fit different recipes \nA workflow set can cross models and/or preprocessors and then resample them en masse.\n\nno_coord_rec <- \n nhl_indicators %>% \n step_rm(starts_with(\"coord\"))\n\nset.seed(9)\n\nnhl_glm_set_res <-\n workflow_set(\n list(`1_no_coord` = no_coord_rec, `2_other` = nhl_other_rec, \n `3_effects` = nhl_effect_rec, `4_angle` = nhl_angle_rec, \n `5_zone` = nhl_zone_rec, `6_bgl` = nhl_behind_rec),\n list(logistic = logistic_reg())\n ) %>%\n workflow_map(fn = \"fit_resamples\", resamples = nhl_val, verbose = TRUE, control = ctrl)" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#monte-carlo-cross-validation", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#monte-carlo-cross-validation", + "title": "4 - Evaluating models", + "section": "Monte Carlo Cross-Validation ", + "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(forested_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6561/2188]> Resample01\n#> 2 <split [6561/2188]> Resample02\n#> 3 <split [6561/2188]> Resample03\n#> 4 <split [6561/2188]> Resample04\n#> 5 <split [6561/2188]> Resample05\n#> 6 <split [6561/2188]> Resample06\n#> 7 <split [6561/2188]> Resample07\n#> 8 <split [6561/2188]> Resample08\n#> 9 <split [6561/2188]> Resample09\n#> 10 <split [6561/2188]> Resample10" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-4", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#your-turn-4", - "title": "5 - Feature engineering", - "section": "Your turn", - "text": "Your turn\n\nCreate a workflow set with 2 or 3 recipes.\n(Consider using recipes we’ve already created.)\nUse workflow_map() to resample the workflow set.\n\n\n\n08:00" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#validation-set", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#validation-set", + "title": "4 - Evaluating models", + "section": "Validation set ", + "text": "Validation set \n\nset.seed(853)\nforested_val_split <- initial_validation_split(forested)\nvalidation_set(forested_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [4264/1421]> validation\n\n\nA validation set is just another type of resample" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#compare-recipes", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#compare-recipes", - "title": "5 - Feature engineering", - "section": "Compare recipes", - "text": "Compare recipes\n\nlibrary(forcats)\ncollect_metrics(nhl_glm_set_res) %>%\n filter(.metric == \"roc_auc\") %>%\n mutate(\n features = gsub(\"_logistic\", \"\", wflow_id), \n features = fct_reorder(features, mean)\n ) %>%\n ggplot(aes(x = mean, y = features)) +\n geom_point(size = 3) +\n labs(y = NULL, x = \"ROC AUC (validation set)\")" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#random-forest-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#random-forest-1", + "title": "4 - Evaluating models", + "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this later!), as long as there are enough trees" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#compare-recipes-1", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#compare-recipes-1", - "title": "5 - Feature engineering", - "section": "Compare recipes", - "text": "Compare recipes" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#create-a-random-forest-model", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#create-a-random-forest-model", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#debugging-a-recipe", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#debugging-a-recipe", - "title": "5 - Feature engineering", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. encoded_players) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#create-a-random-forest-model-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#create-a-random-forest-model-1", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#example", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#example", - "title": "5 - Feature engineering", - "section": "Example", - "text": "Example\n\nnhl_angle_fit <- prep(nhl_angle_rec)\n\ntidy(nhl_angle_fit, number = 1) %>% slice(1:4)\n#> # A tibble: 4 × 4\n#> level value terms id \n#> <chr> <dbl> <chr> <chr> \n#> 1 aaron_ekblad 0.250 shooter lencode_mixed_qBdjK\n#> 2 adam_clendening 0.202 shooter lencode_mixed_qBdjK\n#> 3 adam_cracknell 0.288 shooter lencode_mixed_qBdjK\n#> 4 adam_henrique 0.221 shooter lencode_mixed_qBdjK\n\nbake(nhl_angle_fit, nhl_train %>% slice(1:3), starts_with(\"coord\"), angle, shooter)\n#> # A tibble: 3 × 4\n#> coord_x coord_y angle shooter\n#> <dbl> <dbl> <dbl> <dbl>\n#> 1 67 -37 59.3 0.220\n#> 2 55 -7 11.6 0.302\n#> 3 42 -18 21.0 0.261" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-5", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-5", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#more-on-recipes", - "href": "archive/2022-08-Reykjavik-City/05-feature-engineering.html#more-on-recipes", - "title": "5 - Feature engineering", - "section": "More on recipes", - "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-4", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-4", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nctrl_forested <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, forested_folds, control = ctrl_forested)\ncollect_metrics(rf_res)\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.752 10 0.00469 Preprocessor1_Model1\n#> 2 brier_class binary 0.167 10 0.00317 Preprocessor1_Model1\n#> 3 roc_auc binary 0.757 10 0.0100 Preprocessor1_Model1" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#chicago-l-train-data", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#chicago-l-train-data", - "title": "7 - Case Study on Transportation", - "section": "Chicago L-Train data", - "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game---status-update-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game---status-update-1", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#predictors", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#predictors", - "title": "7 - Case Study on Transportation", - "section": "Predictors", - "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-final-fit", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-final-fit", + "title": "4 - Evaluating models", + "section": "The final fit ", + "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# forested_split has train + test info\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [8749/2188]> train/test split <tibble> <tibble> <tibble> <workflow>" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#l-train-locations", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#l-train-locations", - "title": "7 - Case Study on Transportation", - "section": "L Train Locations", - "text": "L Train Locations" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.761 Preprocessor1_Model1\n#> 2 roc_auc binary 0.761 Preprocessor1_Model1\n#> 3 brier_class binary 0.162 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#your-turn-explore-the-data", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#your-turn-explore-the-data", - "title": "7 - Case Study on Transportation", - "section": "Your turn: Explore the Data", - "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n05:00" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit-1", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit-1", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 2,188 × 7\n#> .pred_class .pred_Yes .pred_No id .row forested .config \n#> <fct> <dbl> <dbl> <chr> <int> <fct> <chr> \n#> 1 Yes 0.964 0.0363 train/test split 4 Yes Preprocessor1…\n#> 2 Yes 0.793 0.207 train/test split 8 Yes Preprocessor1…\n#> 3 Yes 0.815 0.185 train/test split 10 Yes Preprocessor1…\n#> 4 Yes 0.852 0.148 train/test split 19 Yes Preprocessor1…\n#> 5 Yes 0.838 0.162 train/test split 23 Yes Preprocessor1…\n#> 6 Yes 0.552 0.448 train/test split 28 No Preprocessor1…\n#> 7 Yes 0.831 0.169 train/test split 34 Yes Preprocessor1…\n#> 8 Yes 0.614 0.386 train/test split 35 No Preprocessor1…\n#> 9 Yes 0.975 0.0251 train/test split 38 Yes Preprocessor1…\n#> 10 Yes 0.944 0.0561 train/test split 40 Yes Preprocessor1…\n#> # ℹ 2,178 more rows" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#splitting-with-chicago-data", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#splitting-with-chicago-data", - "title": "7 - Case Study on Transportation", - "section": "Splitting with Chicago data ", - "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit-2", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit-2", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 8749 \n#> Number of independent variables: 18 \n#> Mtry: 4 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.1669661\n\n\nUse this for prediction on new data, like for deploying" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#time-series-resampling", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#time-series-resampling", - "title": "7 - Case Study on Transportation", - "section": "Time series resampling", - "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" + "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game", + "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game", + "title": "4 - Evaluating models", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#rolling-forecast-origin-resampling", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#rolling-forecast-origin-resampling", - "title": "7 - Case Study on Transportation", - "section": "Rolling forecast origin resampling", - "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." + "objectID": "archive/2025-08-nyr/slides/intro-06-wrapping-up.html#your-turn", + "href": "archive/2025-08-nyr/slides/intro-06-wrapping-up.html#your-turn", + "title": "6 - Wrapping up", + "section": "Your turn", + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling", - "title": "7 - Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." + "objectID": "archive/2025-08-nyr/slides/intro-06-wrapping-up.html#resources-to-keep-learning", + "href": "archive/2025-08-nyr/slides/intro-06-wrapping-up.html#resources-to-keep-learning", + "title": "6 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://tidymodels.aml4td.org/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-1", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-1", - "title": "7 - Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." + "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", + "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", + "title": "Extras - workflowsets", + "section": "How can we compare multiple model workflows at once?", + "text": "How can we compare multiple model workflows at once?" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-2", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-2", - "title": "7 - Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" + "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set", + "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nwf_set <- workflow_set(list(forested ~ .), list(tree_spec, rf_spec))\nwf_set\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-3", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-3", - "title": "7 - Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" + "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", + "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nwf_set_fit <- wf_set |>\n workflow_map(\"fit_resamples\", resamples = forested_folds)\nwf_set_fit\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-4", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#times-series-resampling-4", - "title": "7 - Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" + "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", + "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nwf_set_fit |>\n rank_results()\n#> # A tibble: 6 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_for… Prepro… accura… 0.753 0.00435 10 formula rand… 1\n#> 2 formula_rand_for… Prepro… brier_… 0.167 0.00316 10 formula rand… 1\n#> 3 formula_rand_for… Prepro… roc_auc 0.757 0.00992 10 formula rand… 1\n#> 4 formula_decision… Prepro… accura… 0.704 0.00653 10 formula deci… 2\n#> 5 formula_decision… Prepro… brier_… 0.214 0.00349 10 formula deci… 2\n#> 6 formula_decision… Prepro… roc_auc 0.692 0.00496 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#our-resampling-object", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#our-resampling-object", - "title": "7 - Case Study on Transportation", - "section": "Our resampling object ", - "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." + "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#your-turn", + "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#your-turn", + "title": "Extras - workflowsets", + "section": "Your turn", + "text": "Your turn\n\nWhen do you think a workflow set would be useful?\nDiscuss with your neighbors!\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#feature-engineering-with-recipes", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#feature-engineering-with-recipes", - "title": "7 - Case Study on Transportation", - "section": "Feature engineering with recipes ", - "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" + "objectID": "slides/advanced-02-model-tuning.html#startup", + "href": "slides/advanced-02-model-tuning.html#startup", + "title": "2 - Model optimization by tuning", + "section": "Startup! ", + "text": "Startup! \n\nlibrary(tidymodels)\nlibrary(probably)\nlibrary(desirability2)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)\n\n# check torch:\nif (torch::torch_is_installed()) {\n library(torch)\n}\n\n# Load our example data for this section\n\"https://raw.githubusercontent.com/tidymodels/\" |> \n paste0(\"workshops/main/slides/class_data.RData\") |> \n url() |> \n load()" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe", - "title": "7 - Case Study on Transportation", - "section": "A recipe", - "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" + "objectID": "slides/advanced-02-model-tuning.html#where-we-are", + "href": "slides/advanced-02-model-tuning.html#where-we-are", + "title": "2 - Model optimization by tuning", + "section": "Where we are ", + "text": "Where we are \n\nset.seed(429)\nsim_split <- initial_split(class_data, prop = 0.75, strata = class)\nsim_train <- training(sim_split)\nsim_test <- testing(sim_split)\n\nset.seed(523)\nsim_rs <- vfold_cv(sim_train, v = 10, strata = class)" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates", - "title": "7 - Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." + "objectID": "slides/advanced-02-model-tuning.html#the-brulee-package", + "href": "slides/advanced-02-model-tuning.html#the-brulee-package", + "title": "2 - Model optimization by tuning", + "section": "The brulee package ", + "text": "The brulee package \nSeveral packages exist for this, but we’ll use the brulee package, which relies on the torch deep learning framework.\n\nLet’s load the package and look at the documentation for the function that we will use:\n\n\n# This might trigger a torch install: \nlibrary(brulee)\n# ?brulee_mlp\n\n(HTML docs are nice too)" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates-1", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates-1", - "title": "7 - Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." + "objectID": "slides/advanced-02-model-tuning.html#notable-arguments-part-1", + "href": "slides/advanced-02-model-tuning.html#notable-arguments-part-1", + "title": "2 - Model optimization by tuning", + "section": "Notable arguments Part 1 ", + "text": "Notable arguments Part 1 \nModel Structure:\n\nhidden_units: the primary way to specify model complexity.\nactivation: the name of the nonlinear function used to connect the predictors to the hidden layer.\n\n\nLoss Function:\n\npenalty: amount of regularization used to prevent overfitting.\nmixture: the proportion of L1 and L2 penalties.\nvalidation: proportion of data to leave out to assess early stopping." }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates-2", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---work-with-dates-2", - "title": "7 - Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points." + "objectID": "slides/advanced-02-model-tuning.html#notable-arguments-part-2", + "href": "slides/advanced-02-model-tuning.html#notable-arguments-part-2", + "title": "2 - Model optimization by tuning", + "section": "Notable arguments Part 2 ", + "text": "Notable arguments Part 2 \nOptimization:\n\noptimizer: the type of gradient-based optimization.\nepochs: how many passes through the entire data set (i.e., iterations).\nstop_iter: number of bad iterations before stopping.\nlearn_rate: how fast does gradient descent move?\nrate_schedule: should the learning rate change over epochs?\nbatch_size: for stochastic gradient descent.\n\n\nThat’s a lot 😩" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---remove-constant-columns", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---remove-constant-columns", - "title": "7 - Case Study on Transportation", - "section": "A recipe - remove constant columns ", - "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE) %>% \n step_zv(all_nominal_predictors())" + "objectID": "slides/advanced-02-model-tuning.html#cost-sensitive-learning", + "href": "slides/advanced-02-model-tuning.html#cost-sensitive-learning", + "title": "2 - Model optimization by tuning", + "section": "Cost-sensitive learning ", + "text": "Cost-sensitive learning \nOne other option: class_weights: amount to upweight the minority class (event) when computing the objective function (cross-entropy).\n\nWe have a moderate class imbalance, and we’ll use this argument to deal with it.\n\nThis will push the minority class probability estimates to be more accurate/calibrated. Overall the model will be less effective; this assumes the minority class is the class of interest." }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---handle-correlations", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#a-recipe---handle-correlations", - "title": "7 - Case Study on Transportation", - "section": "A recipe - handle correlations ", - "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." + "objectID": "slides/advanced-02-model-tuning.html#a-single-model", + "href": "slides/advanced-02-model-tuning.html#a-single-model", + "title": "2 - Model optimization by tuning", + "section": "A single model ", + "text": "A single model \n\nnnet_ex_spec <- \n mlp(hidden_units = 20, penalty = 0.01, learn_rate = 0.005, epochs = 100) |> \n set_engine(\"brulee\", class_weights = 3, stop_iter = 10) |> \n set_mode(\"classification\")\n\nrec <- \n recipe(class ~ ., data = sim_train) |> \n step_normalize(all_numeric_predictors())\n\nnnet_ex_wflow <- workflow(rec, nnet_ex_spec)\n\n# Fit on the first fold's 90% analysis set\nset.seed(147)\nnnet_ex_fit <- fit(nnet_ex_wflow, data = analysis(sim_rs$splits[[1]]))" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#make-some-models", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#make-some-models", - "title": "7 - Case Study on Transportation", - "section": "Make some models ", - "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) %>% set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) %>% \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." + "objectID": "slides/advanced-02-model-tuning.html#did-it-converge", + "href": "slides/advanced-02-model-tuning.html#did-it-converge", + "title": "2 - Model optimization by tuning", + "section": "Did it converge? ", + "text": "Did it converge? \n\n\n\nnnet_ex_fit |>\n # pull out the brulee fit:\n extract_fit_engine() |>\n autoplot()\n\n\n\n\n\n\n\n\n\n\n\nThe y-axis statistics are computed on the held-out predictions at each iteration.\nThe vertical green line shows that early stopping occurred." }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#process-them-on-the-resamples", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#process-them-on-the-resamples", - "title": "7 - Case Study on Transportation", - "section": "Process them on the resamples", - "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set %>% \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" + "objectID": "slides/advanced-02-model-tuning.html#did-it-work", + "href": "slides/advanced-02-model-tuning.html#did-it-work", + "title": "2 - Model optimization by tuning", + "section": "Did it work? ", + "text": "Did it work? \n\nassessment_data <- assessment(sim_rs$splits[[1]])\ncls_mtr <- metric_set(brier_class, roc_auc, sensitivity, specificity)\nholdout_pred <- augment(nnet_ex_fit, assessment_data) \n\n# Performance metrics\nholdout_pred |> cls_mtr(class, estimate = .pred_class, .pred_event)\n#> # A tibble: 4 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.706 \n#> 2 specificity binary 0.970 \n#> 3 brier_class binary 0.0545\n#> 4 roc_auc binary 0.928\n\nKind of?\nIf sensitivity is important, then the model is moderately successful." }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#how-do-the-results-look", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#how-do-the-results-look", - "title": "7 - Case Study on Transportation", - "section": "How do the results look?", - "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 31 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.798 0.104 16 recipe cubis… 1\n#> 2 pca_cubist Preprocessor3_Model3 mae 0.978 0.110 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor4_Model2 mae 0.983 0.122 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model1 mae 0.991 0.127 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor3_Model2 mae 0.991 0.113 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor2_Model2 mae 1.02 0.118 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor1_Model3 mae 1.05 0.134 16 recipe cubis… 7\n#> 8 basic_cubist Preprocessor1_Model8 mae 1.07 0.115 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model7 mae 1.07 0.112 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model6 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 21 more rows" + "objectID": "slides/advanced-02-model-tuning.html#calibration", + "href": "slides/advanced-02-model-tuning.html#calibration", + "title": "2 - Model optimization by tuning", + "section": "Calibration", + "text": "Calibration\nCalibration is a property of individual predictions. A well-calibrated probability occurs in the wild at the same rate as the estimate.\nThe Brier score is the closest we can come to estimating it:\n\\[\nBrier = \\frac{1}{NC}\\sum_{i=1}^N\\sum_{k=1}^C (y_{ik} - \\hat{p}_{ik})^2\n\\]\nZero is best and, for two classes, values above 0.25 are bad.\n\nAML4TD" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#plot-the-results", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#plot-the-results", - "title": "7 - Case Study on Transportation", - "section": "Plot the results ", - "text": "Plot the results \n\nautoplot(chi_res)" + "objectID": "slides/advanced-02-model-tuning.html#calibration-plots", + "href": "slides/advanced-02-model-tuning.html#calibration-plots", + "title": "2 - Model optimization by tuning", + "section": "Calibration Plots", + "text": "Calibration Plots\n\n\n\nholdout_pred |>\n cal_plot_windowed(\n truth = class,\n estimate = .pred_event,\n window_size = 0.2,\n step_size = 0.025\n )\n\n\n😱\nThis is partly due to the small sample size.\n\n\n\n\n\n\n\n\n\n\n\n\nAML4TD" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#pull-out-specific-results", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#pull-out-specific-results", - "title": "7 - Case Study on Transportation", - "section": "Pull out specific results ", - "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n autoplot()" + "objectID": "slides/advanced-02-model-tuning.html#tuning-parameters", + "href": "slides/advanced-02-model-tuning.html#tuning-parameters", + "title": "2 - Model optimization by tuning", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit", - "title": "7 - Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." + "objectID": "slides/advanced-02-model-tuning.html#optimize-tuning-parameters", + "href": "slides/advanced-02-model-tuning.html#optimize-tuning-parameters", + "title": "2 - Model optimization by tuning", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-1", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-1", - "title": "7 - Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "slides/advanced-02-model-tuning.html#tagging-parameters-for-tuning", + "href": "slides/advanced-02-model-tuning.html#tagging-parameters-for-tuning", + "title": "2 - Model optimization by tuning", + "section": "Tagging parameters for tuning ", + "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-2", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-2", - "title": "7 - Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "slides/advanced-02-model-tuning.html#optimizing-the-neural-network", + "href": "slides/advanced-02-model-tuning.html#optimizing-the-neural-network", + "title": "2 - Model optimization by tuning", + "section": "Optimizing the neural network ", + "text": "Optimizing the neural network \n\nnnet_spec <- \n mlp(hidden_units = tune(), penalty = tune(), learn_rate = tune(), \n epochs = 100, activation = tune()) |> \n set_engine(\"brulee\", class_weights = tune(), stop_iter = 10) |> \n set_mode(\"classification\")\n\nnnet_wflow <- workflow(rec, nnet_spec)\nnnet_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: mlp()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 1 Recipe Step\n#> \n#> • step_normalize()\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Single Layer Neural Network Model Specification (classification)\n#> \n#> Main Arguments:\n#> hidden_units = tune()\n#> penalty = tune()\n#> epochs = 100\n#> activation = tune()\n#> learn_rate = tune()\n#> \n#> Engine-Specific Arguments:\n#> class_weights = tune()\n#> stop_iter = 10\n#> \n#> Computational engine: brulee" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-3", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-3", - "title": "7 - Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "slides/advanced-02-model-tuning.html#optimize-tuning-parameters-1", + "href": "slides/advanced-02-model-tuning.html#optimize-tuning-parameters-1", + "title": "2 - Model optimization by tuning", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe two main strategies for optimization are:\n\nGrid search, which tests a pre-defined set of candidate values.\nIterative search, which suggests/estimates new values of candidate parameters to evaluate.\n\n\nWe won’t be discussing iterative search methods in the regular notes. But you can learn more in AML4TD or in the extra slides for this subject." }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-4", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-4", - "title": "7 - Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "slides/advanced-02-model-tuning.html#grid-search", + "href": "slides/advanced-02-model-tuning.html#grid-search", + "title": "2 - Model optimization by tuning", + "section": "Grid search", + "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-5", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#why-choose-just-one-final_fit-5", - "title": "7 - Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "slides/advanced-02-model-tuning.html#grid-search-1", + "href": "slides/advanced-02-model-tuning.html#grid-search-1", + "title": "2 - Model optimization by tuning", + "section": "Grid search", + "text": "Grid search\nIn reality, we would probably sample the space more densely:" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#building-a-model-stack", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#building-a-model-stack", - "title": "7 - Case Study on Transportation", - "section": "Building a model stack ", - "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" + "objectID": "slides/advanced-02-model-tuning.html#iterative-search", + "href": "slides/advanced-02-model-tuning.html#iterative-search", + "title": "2 - Model optimization by tuning", + "section": "Iterative Search", + "text": "Iterative Search\nWe could start with a few points and search the space:" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#start-the-stack-and-add-members", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#start-the-stack-and-add-members", - "title": "7 - Case Study on Transportation", - "section": "Start the stack and add members ", - "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() %>% \n add_candidates(chi_res)" + "objectID": "slides/advanced-02-model-tuning.html#parameters", + "href": "slides/advanced-02-model-tuning.html#parameters", + "title": "2 - Model optimization by tuning", + "section": "Parameters", + "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc.).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nnnet_param <- \n nnet_wflow |> \n extract_parameter_set_dials() \nnnet_param \n#> Collection of 5 parameters for tuning\n#> \n#> identifier type object\n#> hidden_units hidden_units nparam[+]\n#> penalty penalty nparam[+]\n#> activation activation dparam[+]\n#> learn_rate learn_rate nparam[+]\n#> class_weights class_weights nparam[+]\n#>" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#estimate-weights-for-each-candidate", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#estimate-weights-for-each-candidate", - "title": "7 - Case Study on Transportation", - "section": "Estimate weights for each candidate ", - "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack)\n\nchi_stack_res\n#> # A tibble: 5 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.343\n#> 2 pca_cubist_3_2 cubist_rules 0.236\n#> 3 basic_cubist_1_4 cubist_rules 0.189\n#> 4 pca_lm_4_1 linear_reg 0.163\n#> 5 pca_cubist_3_3 cubist_rules 0.109" + "objectID": "slides/advanced-02-model-tuning.html#different-types-of-grids", + "href": "slides/advanced-02-model-tuning.html#different-types-of-grids", + "title": "2 - Model optimization by tuning", + "section": "Different types of grids ", + "text": "Different types of grids \n\n\n\n\n\n\n\n\n\nSpace-filling designs (SFD) attempt to cover the parameter space without redundant candidates. We recommend these the most, and they are the default." }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#how-did-it-do", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#how-did-it-do", - "title": "7 - Case Study on Transportation", - "section": "How did it do? ", - "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" + "objectID": "slides/advanced-02-model-tuning.html#extract-and-update-parameters", + "href": "slides/advanced-02-model-tuning.html#extract-and-update-parameters", + "title": "2 - Model optimization by tuning", + "section": "Extract and update parameters ", + "text": "Extract and update parameters \n\nnnet_param <- \n nnet_param |> \n update(class_weights = class_weights(c(1, 50)))\n \n nnet_param \n#> Collection of 5 parameters for tuning\n#> \n#> identifier type object\n#> hidden_units hidden_units nparam[+]\n#> penalty penalty nparam[+]\n#> activation activation dparam[+]\n#> learn_rate learn_rate nparam[+]\n#> class_weights class_weights nparam[+]\n#>" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#what-does-it-use", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#what-does-it-use", - "title": "7 - Case Study on Transportation", - "section": "What does it use? ", - "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" + "objectID": "slides/advanced-02-model-tuning.html#grid-search-3", + "href": "slides/advanced-02-model-tuning.html#grid-search-3", + "title": "2 - Model optimization by tuning", + "section": "Grid Search ", + "text": "Grid Search \n\nctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nset.seed(12)\nnnet_res <-\n nnet_wflow |>\n tune_grid(\n resamples = sim_rs,\n grid = 25,\n # The options below are not required by default\n param_info = nnet_param, \n control = ctrl,\n metrics = cls_mtr\n )\n\n\n\n✋ maybe don’t run this just yet…\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#fit-the-required-candidate-models", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#fit-the-required-candidate-models", - "title": "7 - Case Study on Transportation", - "section": "Fit the required candidate models", - "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" + "objectID": "slides/advanced-02-model-tuning.html#running-in-parallel", + "href": "slides/advanced-02-model-tuning.html#running-in-parallel", + "title": "2 - Model optimization by tuning", + "section": "Running in parallel", + "text": "Running in parallel\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use the future or mirai packages to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\n\n\n\n\n\nlibrary(future)\nplan(multisession, workers = cores)\n\n# Now call `tune_grid()`!\n\n\n\nlibrary(mirai)\ndaemons(cores)\n\n# Now call `tune_grid()`!" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#the-test-set-best-cubist-model", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#the-test-set-best-cubist-model", - "title": "7 - Case Study on Transportation", - "section": "The test set: best Cubist model ", - "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n select_best()\n\ncubist_res <- \n chi_res %>% \n extract_workflow(\"pca_cubist\") %>% \n finalize_workflow(best_cubist) %>% \n last_fit(split = chi_split, metrics = metric_set(mae))" + "objectID": "slides/advanced-02-model-tuning.html#distributing-tasks", + "href": "slides/advanced-02-model-tuning.html#distributing-tasks", + "title": "2 - Model optimization by tuning", + "section": "Distributing tasks", + "text": "Distributing tasks\nWhen only tuning the model:" }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#the-test-set-stack-ensemble", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#the-test-set-stack-ensemble", - "title": "7 - Case Study on Transportation", - "section": "The test set: stack ensemble", - "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) %>% \n bind_cols(chi_test)" + "objectID": "slides/advanced-02-model-tuning.html#running-in-parallel-1", + "href": "slides/advanced-02-model-tuning.html#running-in-parallel-1", + "title": "2 - Model optimization by tuning", + "section": "Running in parallel", + "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#compare-the-results", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#compare-the-results", - "title": "7 - Case Study on Transportation", - "section": "Compare the results ", - "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.670 Preprocessor1_Model1\n\nstack_pred %>% mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.689" + "objectID": "slides/advanced-02-model-tuning.html#running-in-parallel-2", + "href": "slides/advanced-02-model-tuning.html#running-in-parallel-2", + "title": "2 - Model optimization by tuning", + "section": "Running in parallel", + "text": "Running in parallel\nWe’ll use mirai as our parallel backend for our notes.\n\nUsing 10 cores with mirai, time to execute the previous tune_grid() call was reduced from 256s to 45s, a speed-up of 5.7-fold.\n\nThe next slide deck (on racing) includes more examples of speed-ups for a different model." }, { - "objectID": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#plot-the-test-set", - "href": "archive/2022-08-Reykjavik-City/07-transit-case-study.html#plot-the-test-set", - "title": "7 - Case Study on Transportation", - "section": "Plot the test set ", - "text": "Plot the test set \n\n\ncubist_res %>% \n collect_predictions() %>% \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" + "objectID": "slides/advanced-02-model-tuning.html#grid-search-4", + "href": "slides/advanced-02-model-tuning.html#grid-search-4", + "title": "2 - Model optimization by tuning", + "section": "Grid Search ", + "text": "Grid Search \n\nnnet_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [1348/151]> Fold01 <tibble [100 × 9]> <tibble [0 × 4]> <tibble> \n#> 2 <split [1349/150]> Fold02 <tibble [100 × 9]> <tibble [0 × 4]> <tibble> \n#> 3 <split [1349/150]> Fold03 <tibble [100 × 9]> <tibble [0 × 4]> <tibble> \n#> 4 <split [1349/150]> Fold04 <tibble [100 × 9]> <tibble [0 × 4]> <tibble> \n#> 5 <split [1349/150]> Fold05 <tibble [100 × 9]> <tibble [1 × 4]> <tibble> \n#> 6 <split [1349/150]> Fold06 <tibble [100 × 9]> <tibble [1 × 4]> <tibble> \n#> 7 <split [1349/150]> Fold07 <tibble [100 × 9]> <tibble [1 × 4]> <tibble> \n#> 8 <split [1349/150]> Fold08 <tibble [100 × 9]> <tibble [0 × 4]> <tibble> \n#> 9 <split [1350/149]> Fold09 <tibble [100 × 9]> <tibble [0 × 4]> <tibble> \n#> 10 <split [1350/149]> Fold10 <tibble [100 × 9]> <tibble [1 × 4]> <tibble> \n#> \n#> There were issues with some computations:\n#> \n#> - Warning(s) x1: Early stopping occurred at epoch 2 due to numerical overflow of t...\n#> - Warning(s) x1: Early stopping occurred at epoch 3 due to numerical overflow of t...\n#> - Warning(s) x2: Early stopping occurred at epoch 5 due to numerical overflow of t...\n#> \n#> Run `show_notes(.Last.tune.result)` for more information." }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html", - "href": "archive/2022-08-Reykjavik-City/annotations.html", - "title": "Annotations", - "section": "", - "text": "This page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources.\n\n\n\nThis is a pretty complex data usage scheme. That is mostly because of the validation set. In every other case, the situation is much more simple.\nThe important point here is that: tidymodels does most of this work for you. In other words, you don’t have to directly specify which data are being used where.\nIn a later section, we will talk about methods of resampling. These methods are like repeated validation sets. As an example, the popular 10-fold cross-validation method is one such type of resampling. Validation sets are special cases of resampling where there is a single “resample”.\nMost types of resampling use multiple hold-out sets of samples from the training set. In those cases, a diagram for data usage here would look like\n\n\n\n\n\n\n\n\n\nIn this case there is just “testing” and “training”. Once the final model is determined, the entire training set is used for the last fit.\nThis is the process that will be used for the tree frog data." + "objectID": "slides/advanced-02-model-tuning.html#grid-results", + "href": "slides/advanced-02-model-tuning.html#grid-results", + "title": "2 - Model optimization by tuning", + "section": "Grid results ", + "text": "Grid results \n\nautoplot(nnet_res)" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#section", - "href": "archive/2022-08-Reykjavik-City/annotations.html#section", - "title": "Annotations", - "section": "", - "text": "This page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." + "objectID": "slides/advanced-02-model-tuning.html#brier-results", + "href": "slides/advanced-02-model-tuning.html#brier-results", + "title": "2 - Model optimization by tuning", + "section": "Brier results ", + "text": "Brier results \n\nautoplot(nnet_res, metric = \"brier_class\") + \n facet_grid(. ~ name, scale = \"free_x\") + \n lims(y = c(0.04, 0.22)) # no tanh" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#finalize-and-verify", - "href": "archive/2022-08-Reykjavik-City/annotations.html#finalize-and-verify", - "title": "Annotations", - "section": "", - "text": "This is a pretty complex data usage scheme. That is mostly because of the validation set. In every other case, the situation is much more simple.\nThe important point here is that: tidymodels does most of this work for you. In other words, you don’t have to directly specify which data are being used where.\nIn a later section, we will talk about methods of resampling. These methods are like repeated validation sets. As an example, the popular 10-fold cross-validation method is one such type of resampling. Validation sets are special cases of resampling where there is a single “resample”.\nMost types of resampling use multiple hold-out sets of samples from the training set. In those cases, a diagram for data usage here would look like\n\n\n\n\n\n\n\n\n\nIn this case there is just “testing” and “training”. Once the final model is determined, the entire training set is used for the last fit.\nThis is the process that will be used for the tree frog data." + "objectID": "slides/advanced-02-model-tuning.html#roc-curve-results", + "href": "slides/advanced-02-model-tuning.html#roc-curve-results", + "title": "2 - Model optimization by tuning", + "section": "ROC curve results ", + "text": "ROC curve results \n\nautoplot(nnet_res, metric = \"roc_auc\") + \n facet_grid(. ~ name, scale = \"free_x\") + \n lims(y = c(0.83, 0.97)) # no tanh" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#data-splitting-and-spending", - "href": "archive/2022-08-Reykjavik-City/annotations.html#data-splitting-and-spending", - "title": "Annotations", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n\n#> [1] 0.2655087 0.3721239 0.5728534\n\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" + "objectID": "slides/advanced-02-model-tuning.html#sensitivityspecificity-results", + "href": "slides/advanced-02-model-tuning.html#sensitivityspecificity-results", + "title": "2 - Model optimization by tuning", + "section": "Sensitivity/Specificity results ", + "text": "Sensitivity/Specificity results \n\nautoplot(nnet_res, metric = c(\"sensitivity\", \"specificity\"))" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#what-is-wrong-with-this", - "href": "archive/2022-08-Reykjavik-City/annotations.html#what-is-wrong-with-this", - "title": "Annotations", - "section": "What is wrong with this?", - "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" + "objectID": "slides/advanced-02-model-tuning.html#grid-results-1", + "href": "slides/advanced-02-model-tuning.html#grid-results-1", + "title": "2 - Model optimization by tuning", + "section": "Grid results", + "text": "Grid results\n\ntanh activation 👎👎\nAs class weight ⬆️:\n\nsensitivity ⬆️\nspecificity ⬇️⬇️\nBrier: ⬆️\nROC AUC: 🤷" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#where-are-the-fitted-models", - "href": "archive/2022-08-Reykjavik-City/annotations.html#where-are-the-fitted-models", - "title": "Annotations", - "section": "Where are the fitted models?", - "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" + "objectID": "slides/advanced-02-model-tuning.html#tuning-results", + "href": "slides/advanced-02-model-tuning.html#tuning-results", + "title": "2 - Model optimization by tuning", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(nnet_res) |> \n relocate(.metric, mean) \n#> # A tibble: 100 × 11\n#> .metric mean hidden_units penalty activation learn_rate class_weights\n#> <chr> <dbl> <int> <dbl> <chr> <dbl> <dbl>\n#> 1 brier_class 0.135 2 0.00000147 elu 0.0962 17.3\n#> 2 roc_auc 0.887 2 0.00000147 elu 0.0962 17.3\n#> 3 sensitivity 0.875 2 0.00000147 elu 0.0962 17.3\n#> 4 specificity 0.778 2 0.00000147 elu 0.0962 17.3\n#> 5 brier_class 0.134 4 0.000464 tanhshrink 0.0736 31.6\n#> 6 roc_auc 0.924 4 0.000464 tanhshrink 0.0736 31.6\n#> 7 sensitivity 0.863 4 0.000464 tanhshrink 0.0736 31.6\n#> 8 specificity 0.802 4 0.000464 tanhshrink 0.0736 31.6\n#> 9 brier_class 0.203 6 0.00000383 relu 0.00224 41.8\n#> 10 roc_auc 0.880 6 0.00000383 relu 0.00224 41.8\n#> # ℹ 90 more rows\n#> # ℹ 4 more variables: .estimator <chr>, n <int>, std_err <dbl>, .config <chr>" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#the-final-fit", - "href": "archive/2022-08-Reykjavik-City/annotations.html#the-final-fit", - "title": "Annotations", - "section": "The final fit", - "text": "The final fit\nSince our data spending scheme created the resamples from the training set, last_fit() will use all of the training data to fit the final workflow.\nAs shown in the Whole Game slides, there is a slightly different scheme used when we have a validation set (instead of multiple resamples like 10-fold CV)." + "objectID": "slides/advanced-02-model-tuning.html#tuning-results-1", + "href": "slides/advanced-02-model-tuning.html#tuning-results-1", + "title": "2 - Model optimization by tuning", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(nnet_res, summarize = FALSE) |> \n relocate(.metric, .estimate)\n#> # A tibble: 1,000 × 10\n#> .metric .estimate id hidden_units penalty activation learn_rate\n#> <chr> <dbl> <chr> <int> <dbl> <chr> <dbl>\n#> 1 sensitivity 1 Fold01 2 0.00000147 elu 0.0962\n#> 2 specificity 0.851 Fold01 2 0.00000147 elu 0.0962\n#> 3 brier_class 0.0886 Fold01 2 0.00000147 elu 0.0962\n#> 4 roc_auc 0.972 Fold01 2 0.00000147 elu 0.0962\n#> 5 sensitivity 0.941 Fold02 2 0.00000147 elu 0.0962\n#> 6 specificity 0.887 Fold02 2 0.00000147 elu 0.0962\n#> 7 brier_class 0.0847 Fold02 2 0.00000147 elu 0.0962\n#> 8 roc_auc 0.974 Fold02 2 0.00000147 elu 0.0962\n#> 9 sensitivity 0.882 Fold03 2 0.00000147 elu 0.0962\n#> 10 specificity 0.872 Fold03 2 0.00000147 elu 0.0962\n#> # ℹ 990 more rows\n#> # ℹ 3 more variables: class_weights <dbl>, .estimator <chr>, .config <chr>" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#using-a-workflow", - "href": "archive/2022-08-Reykjavik-City/annotations.html#using-a-workflow", - "title": "Annotations", - "section": "Using a workflow", - "text": "Using a workflow\nWhat’s going on with the\n\nprediction from a rank-deficient fit may be misleading\n\nwarnings?\nFor linear regression, a computation is used called matrix inversion. The matrix in question is called the “model matrix” and it contains the predictor set for the training data.\nMatrix inversion can fail if two or more columns:\n\nare identical, or\nadd up to some other column.\n\nThese situations are called linear dependencies.\nWhen this happens, lm() is pretty tolerant. It does not fail but does not compute regression coefficients for a minimal number of predictors involved in the dependency (and issues the warning above).\nFor these data, there is this dependency:\nshooter_type_{level} = shooter_{level}\nHere is what is happening: since the player shooting only ever plays a single position, their indicators sum up (row-wide) to the same data as the sum of the shooter type indicators.\nIn other words, if you know the player’s name, you know their position too. This is a perfect redundency in the data.\nThe way to avoid this problem is to use step_lincomb(all_numeric_predictors()) in the recipe. This step removes the minimum number of columns to avoid the issue.\nJust in case you ever want to figure out what the specific issues are, this code might help:\n\n# Get the exact data set used to fit the model. For a recipe, that is:\nprocessed_data <- \n nhl_indicators %>% \n prep() %>% \n bake(new_data = NULL, all_predictors())\n\n# Let's capture their names:\npred_names <- names(processed_data)\n\n# caret has a function to determine the variables involved in each dependency\nissues <- caret::findLinearCombos(processed_data)\n# Convert the column index to names:\nissues_vars <- purrr::map(issues$linearCombos, ~ pred_names[.x])\n\n# caret proposes removing these columns to get rid of the issue. The choice\n# is pretty arbitrary: \nremoved_names <- pred_names[issues$remove]\n\ntl;dr\nLinear regression detects some redundancies in the predictor set. We can ignore the warnings since lm() can deal with it or use step_lincomb() to avoid the warnings." + "objectID": "slides/advanced-02-model-tuning.html#choose-a-parameter-combination", + "href": "slides/advanced-02-model-tuning.html#choose-a-parameter-combination", + "title": "2 - Model optimization by tuning", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \n\nshow_best(nnet_res, metric = \"brier_class\") |> \n relocate(.metric, mean) \n#> # A tibble: 5 × 11\n#> .metric mean hidden_units penalty activation learn_rate class_weights\n#> <chr> <dbl> <int> <dbl> <chr> <dbl> <dbl>\n#> 1 brier_class 0.0496 36 0.0000261 elu 0.0562 3.04\n#> 2 brier_class 0.0705 14 0.000178 tanhshrink 0.0329 1 \n#> 3 brier_class 0.0720 18 0.0562 log_sigmoid 0.631 19.4 \n#> 4 brier_class 0.0962 40 0.0215 tanh 0.00293 7.12\n#> 5 brier_class 0.100 44 0.383 log_sigmoid 0.0430 27.5 \n#> # ℹ 4 more variables: .estimator <chr>, n <int>, std_err <dbl>, .config <chr>" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#per-player-statistics", - "href": "archive/2022-08-Reykjavik-City/annotations.html#per-player-statistics", - "title": "Annotations", - "section": "Per-player statistics", - "text": "Per-player statistics\nThe effect encoding method essentially takes the effect of a variable, like player, and makes a data column for that effect. In our example, the ability of a player to have an on-goal shot is quantified by a model and then added as a data column to be used in the model.\nSuppose NHL rookie Max has a single shot in the data and it was on goal. If we used a naive estimate for Max’s effect, the model is being told that Max should have a 100% chance of being on goal. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned player Davis, who has taken 250 shots and 75% of these were on goal. Davis’s proportion is more predictive because it is estimated with better data (i.e., more total shots). Partial pooling leverages the entire data set and can borrow strength from all of the players. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a player’s data is of good quality, the partial pooling effect estimate is closer to the raw proportion. Max’s data is not great and is “shrunk” towards the center of the overall on goal proportion. Since there is so little known about Max’s shot history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nIf the outcome were numeric, the effect would be the mean of the outcome per player. In this case, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" + "objectID": "slides/advanced-02-model-tuning.html#choose-a-parameter-combination-1", + "href": "slides/advanced-02-model-tuning.html#choose-a-parameter-combination-1", + "title": "2 - Model optimization by tuning", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nnnet_best <- select_best(nnet_res, metric = \"brier_class\")\nnnet_best\n#> # A tibble: 1 × 6\n#> hidden_units penalty activation learn_rate class_weights .config \n#> <int> <dbl> <chr> <dbl> <dbl> <chr> \n#> 1 36 0.0000261 elu 0.0562 3.04 pre0_mod18_post0\n\ncollect_metrics(nnet_res) |> \n inner_join(nnet_best) |> \n select(.metric, mean)\n#> # A tibble: 4 × 2\n#> .metric mean\n#> <chr> <dbl>\n#> 1 brier_class 0.0496\n#> 2 roc_auc 0.965 \n#> 3 sensitivity 0.828 \n#> 4 specificity 0.950" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#player-effects", - "href": "archive/2022-08-Reykjavik-City/annotations.html#player-effects", - "title": "Annotations", - "section": "Player effects", - "text": "Player effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for players with small sample sizes. It can’t correct for improper data usage or data leakage though." + "objectID": "slides/advanced-02-model-tuning.html#checking-approximate-calibration", + "href": "slides/advanced-02-model-tuning.html#checking-approximate-calibration", + "title": "2 - Model optimization by tuning", + "section": "Checking (Approximate) Calibration ", + "text": "Checking (Approximate) Calibration \n\n\nnnet_holdout_pred <-\n nnet_res |> \n collect_predictions(\n parameters = nnet_best\n )\n\nnnet_holdout_pred |>\n cal_plot_windowed(\n truth = class,\n estimate = .pred_event,\n window_size = 0.2,\n step_size = 0.025,\n )\n\n\n\n\n\n\n\n\n\n\nThis plot pools the out-of-sample predictions. Our Brier estimate is the mean of 10 data sets.\nThe curve is approximate for that reason." }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#angle", - "href": "archive/2022-08-Reykjavik-City/annotations.html#angle", - "title": "Annotations", - "section": "Angle", - "text": "Angle\nAbout geometry…\nThe coordinates for the rink are centered at (0, 0) and the goal lines are both 89 ft from center. The center of the goal on the left is at (-89, 0) and the right-hand goal is centered at (89, 0).\nFor angle to center of the goal mouth, the formula is\n\\[a = \\tan^{-1}\\left(\\frac{y}{x}\\right)\\] This is in radian units and we can convert to degrees using\n\\[a = \\frac{180}{\\pi}\\tan^{-1}\\left(\\frac{y}{x}\\right)\\] For the angle to the goal, we need to alter \\(x\\) and use x* = (89 - abs(coord_x)) instead." + "objectID": "slides/advanced-02-model-tuning.html#multiple-goals", + "href": "slides/advanced-02-model-tuning.html#multiple-goals", + "title": "2 - Model optimization by tuning", + "section": "Multiple goals", + "text": "Multiple goals\nOptimizing the Brier score optimizes for accuracy and calibration of the class probabilities.\n\nThat’s counter-productive to finding the rare class “event” events.\n\nCan we find a way to optimize multiple metrics at once?\n\nThere are a few ways, and we’ll focus on desirability functions." }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#update-parameter-ranges", - "href": "archive/2022-08-Reykjavik-City/annotations.html#update-parameter-ranges", - "title": "Annotations", - "section": "Update parameter ranges", - "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nIn our case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_ns(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n\n#> # A tibble: 1 × 5\n#> name call_info source component component_id\n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_ns ns_P1Tjg\n\nns_tunable$call_info\n\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 1 15" + "objectID": "slides/advanced-02-model-tuning.html#desirability-functions", + "href": "slides/advanced-02-model-tuning.html#desirability-functions", + "title": "2 - Model optimization by tuning", + "section": "Desirability functions", + "text": "Desirability functions\nWe create simple functions to translate our variable to [0, 1]; 1.0 is most desirable.\n\nWe can combine them with a geometric mean." }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#boosted-tree-tuning-parameters", - "href": "archive/2022-08-Reykjavik-City/annotations.html#boosted-tree-tuning-parameters", - "title": "Annotations", - "section": "Boosted tree tuning parameters", - "text": "Boosted tree tuning parameters\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the xgboost package.\nparsnip has its own wrapper around (xgboost::xgb.train()) called xgb_train(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the xgboost entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for xgboost)." + "objectID": "slides/advanced-02-model-tuning.html#multimetric-optimization", + "href": "slides/advanced-02-model-tuning.html#multimetric-optimization", + "title": "2 - Model optimization by tuning", + "section": "Multimetric optimization ", + "text": "Multimetric optimization \n\nshow_best_desirability(\n nnet_res, \n maximize(sensitivity),\n minimize(brier_class),\n constrain(specificity, low = 0.8, high = 1.0)\n) |> \n relocate(class_weights, sensitivity, specificity, brier_class, .d_overall) \n#> # A tibble: 5 × 14\n#> class_weights sensitivity specificity brier_class .d_overall hidden_units\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <int>\n#> 1 48.0 0.947 0.811 0.127 0.926 34\n#> 2 33.7 0.941 0.805 0.130 0.919 32\n#> 3 25.5 0.899 0.850 0.101 0.918 28\n#> 4 29.6 0.917 0.830 0.117 0.915 38\n#> 5 19.4 0.852 0.914 0.0720 0.907 18\n#> # ℹ 8 more variables: penalty <dbl>, activation <chr>, learn_rate <dbl>,\n#> # .config <chr>, roc_auc <dbl>, .d_max_sensitivity <dbl>,\n#> # .d_min_brier_class <dbl>, .d_box_specificity <dbl>" }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#the-final-fit-to-the-nhl-data", - "href": "archive/2022-08-Reykjavik-City/annotations.html#the-final-fit-to-the-nhl-data", - "title": "Annotations", - "section": "The final fit to the NHL data", - "text": "The final fit to the NHL data\nRecall that last_fit() uses the objects produced by initial_split() to determine what data are used for the final model fit and which are used as the test set.\nFor the validation set, last_fit() will use the non-testing data to create the final model fit. This includes the training and validation set.\nThere is no agreement in the community on whether this is the best approach or if we should just use the training set. There are good arguments either way.\nIf you only want to use the training set for the final model, you can do this via:\n\ntraining_data <- nhl_val$splits[[1]] %>% analysis()\n\n# Use `fit()` to train the model on just the training set\nfinal_glm_spline_wflow <- \n glm_spline_wflow %>% \n fit(data = training_data)\n\n# Create test set predictions\ntest_set_pred <- augment(final_glm_spline_wflow, nhl_test)\n\n# Setup and compute the test set metrics\ncls_metrics <- metric_set(roc_auc, accuracy)\n\ntest_res <- \n test_set_pred %>% \n cls_metrics(on_goal, estimate = .pred_class, .pred_yes)\ntest_res" + "objectID": "slides/advanced-02-model-tuning.html#however", + "href": "slides/advanced-02-model-tuning.html#however", + "title": "2 - Model optimization by tuning", + "section": "However… ", + "text": "However… \n\n\nmore_sens <-\n select_best_desirability(\n nnet_res,\n maximize(sensitivity),\n minimize(brier_class),\n constrain(specificity, low = 0.8, high = 1.0)\n )\n\nnnet_res |>\n collect_predictions(\n parameters = more_sens\n ) |>\n cal_plot_windowed(\n truth = class,\n estimate = .pred_event,\n window_size = 0.2,\n step_size = 0.025,\n )\n\n\n\n\n\n\n\n\n\n\nThe curve hugging the bottom of the plot means that we are drastically overestimating the probability of the “event” class relative to how often it occurs “in the wild.”\nThat’s the point of cost-sensitive learning, but it’s bad for calibration." }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#explain-yourself", - "href": "archive/2022-08-Reykjavik-City/annotations.html#explain-yourself", - "title": "Annotations", - "section": "Explain yourself", - "text": "Explain yourself\nSome other resources:\n\nTMwR chapter Explaining Models and Predictions\nExplanatory Model Analysis book\nInterpretable Machine Learning book\nDefinitions, methods, and applications in interpretable machine learning" + "objectID": "slides/advanced-02-model-tuning.html#conflicting-goals", + "href": "slides/advanced-02-model-tuning.html#conflicting-goals", + "title": "2 - Model optimization by tuning", + "section": "Conflicting goals", + "text": "Conflicting goals\nThe problem here is that we are biasing the probability estimates so that we can predict more data to be the rare “event” class using a default probability cutoff of 1/2.\n\nThat is compromising the overall model fit; our probabilities are not accurate.\n\nIf we don’t use the class probability estimates, this is fine.\n\n\nIn the postprocessing slides, we’ll examine an alternative approach that involves different kinds of tradeoffs." }, { - "objectID": "archive/2022-08-Reykjavik-City/annotations.html#a-tidymodels-explainer", - "href": "archive/2022-08-Reykjavik-City/annotations.html#a-tidymodels-explainer", - "title": "Annotations", - "section": "A tidymodels explainer", - "text": "A tidymodels explainer\nFor our example, the angle was an original predictor. Recall that we made spline terms from this predictor, so there are derived features such as angle_ns_1 and so on.\nOriginal versus derived doesn’t affect local explainers since we are focused on a single prediction.\nFor global explainers, we should decide between:\n\nexplaining the overall affect of angle (lumping all its features into one importance score), or\nexplaining the effect of each term in the model (including angle_ns_1 and so on).\n\nThe choice depends on what you want. For example, if we have an original date predictor and make features for month and year, is it more informative to know if date is important (overall) or exactly how the date is important? You might want to look at it both ways." + "objectID": "slides/advanced-02-model-tuning.html#fitting-a-workflow", + "href": "slides/advanced-02-model-tuning.html#fitting-a-workflow", + "title": "2 - Model optimization by tuning", + "section": "Fitting a workflow ", + "text": "Fitting a workflow \nLet’s say that we want to train the model on the “best” parameter estimates.\nWe can use a tibble of tuning parameters and splice them into the workflow in place of tune():\n\n\n\nset.seed(398)\nnnet_sens_fit <-\n nnet_wflow |>\n finalize_workflow(more_sens) |>\n fit(sim_train)\n\n\n\nnnet_sens_fit |> extract_fit_engine()\n#> Multilayer perceptron\n#> \n#> relu activation,\n#> 34 hidden units,\n#> 1,124 model parameters\n#> 1,499 samples, 30 features, 2 classes \n#> class weights event=47.95833, no_event= 1.00000 \n#> weight decay: 6.812921e-10 \n#> dropout proportion: 0 \n#> batch size: 1350 \n#> learn rate: 0.006556418 \n#> validation loss after 13 epochs: 0.0969" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#who-are-you", - "href": "archive/2023-07-nyr/01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML" + "objectID": "slides/advanced-02-model-tuning.html#ordering-off-of-the-menu", + "href": "slides/advanced-02-model-tuning.html#ordering-off-of-the-menu", + "title": "2 - Model optimization by tuning", + "section": "Ordering off of the menu ", + "text": "Ordering off of the menu \nIf we want to choose from the numerically best for an existing metric, there is a simpler function:\n\n\n\nset.seed(398)\nmlp_brier_fit <-\n nnet_res |>\n fit_best(metric = \"brier_class\")\n\n\n\nmlp_brier_fit |> extract_fit_engine()\n#> Multilayer perceptron\n#> \n#> elu activation,\n#> 36 hidden units,\n#> 1,190 model parameters\n#> 1,499 samples, 30 features, 2 classes \n#> class weights event=3.041667, no_event=1.000000 \n#> weight decay: 2.610157e-05 \n#> dropout proportion: 0 \n#> batch size: 1350 \n#> learn rate: 0.05623413 \n#> validation loss after 2 epochs: 0.304" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#who-are-tidymodels", - "href": "archive/2023-07-nyr/01-introduction.html#who-are-tidymodels", - "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "slides/advanced-02-model-tuning.html#extracting-results", + "href": "slides/advanced-02-model-tuning.html#extracting-results", + "title": "2 - Model optimization by tuning", + "section": "Extracting Results ", + "text": "Extracting Results \nIf we want to know about the resampled workflow, we can write a function that can return information from tune_grid().\n\nFor example, this one can save the optimization process results:\n\n\n\nextract_iter_hist <- function(wflow) {\n require(tidymodels)\n require(tibble)\n wflow |>\n extract_fit_engine() |>\n pluck(\"loss\") |>\n as_tibble_col(\"loss\") |>\n mutate(epoch = row_number() - 1)\n}\n\n\n\nmlp_brier_fit |> \n extract_iter_hist() |> \n slice(1:5)\n#> # A tibble: 5 × 2\n#> loss epoch\n#> <dbl> <dbl>\n#> 1 0.696 0\n#> 2 0.304 1\n#> 3 0.217 2\n#> 4 0.316 3\n#> 5 0.619 4" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#asking-for-help", - "href": "archive/2023-07-nyr/01-introduction.html#asking-for-help", - "title": "1 - Introduction", - "section": "Asking for help", - "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" + "objectID": "slides/advanced-02-model-tuning.html#your-turn", + "href": "slides/advanced-02-model-tuning.html#your-turn", + "title": "2 - Model optimization by tuning", + "section": "Your turn", + "text": "Your turn\n\nRead the docs for control_grid(), specifically the extract option.\nCreate a control object that extracts the iteration history.\nRe-run tune_grid() with the same grid (or fewer grid points) and use the control option with the extraction function.\nAfterward use collect_extracts() to get the results.\nPlot the iteration history for one or more grid points, coloring by the resample id.\n\nParallel processing will make this go more quickly.\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#section", - "href": "archive/2023-07-nyr/01-introduction.html#section", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "slides/advanced-02-model-tuning.html#whats-next", + "href": "slides/advanced-02-model-tuning.html#whats-next", + "title": "2 - Model optimization by tuning", + "section": "What’s next ", + "text": "What’s next \nLet’s look at racing: an old variation of grid search that adaptively processes the grid.\n\nIf we are\n\ninitially screening many models/preprocessors/postprocessors and/or\nhave a large grid\n\nThis can lead to remarkable speed-ups." }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#tentative-plan-for-this-workshop", - "href": "archive/2023-07-nyr/01-introduction.html#tentative-plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Tentative plan for this workshop", - "text": "Tentative plan for this workshop\n\n\n\nToday:\n\nYour data budget\nWhat makes a model\nEvaluating models\n\n\n\n\nTomorrow:\n\nFeature engineering\nTuning hyperparameters\nRacing methods\nIterative search methods" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#getting-set-up", + "href": "slides/advanced-04-feature-engineering-part-one.html#getting-set-up", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Getting set up ", + "text": "Getting set up \n\nlibrary(tidymodels)\nlibrary(embed)\nlibrary(extrasteps)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)\n\n# Load our example data for this section\n\"https://github.com/tidymodels/workshops/raw/refs/heads/2025-GMOFETML/slides/leaf_data.RData\" |> \n url() |> \n load()" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#section-1", - "href": "archive/2023-07-nyr/01-introduction.html#section-1", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋\n\nCheck Slack (#ml-ws-2023) for an RStudio Cloud link." + "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set", + "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Leaf data set", + "text": "Leaf data set\nSlightly modified version of modeldata::leaf_id_flavia.\n\nglimpse(leaf_data)\n#> Rows: 1,907\n#> Columns: 55\n#> $ species <fct> chinese_redbud, chinese_redbud, chinese_red…\n#> $ apex <fct> none, none, none, none, none, none, none, n…\n#> $ base <fct> none, none, none, none, none, none, none, n…\n#> $ shape <fct> heart_shape, heart_shape, heart_shape, hear…\n#> $ edge <chr> \"smooth\", \"smooth\", \"smooth\", \"smooth\", \"sm…\n#> $ outlying_polar <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ skewed_polar <dbl> 0.4610066, 0.4747877, 0.5356883, 0.4771338,…\n#> $ clumpy_polar <dbl> 0.006479784, 0.011100482, 0.017317486, 0.01…\n#> $ sparse_polar <dbl> 0.01449941, 0.01451566, 0.02953578, 0.01425…\n#> $ striated_polar <dbl> 0.9788360, 0.9797980, 0.6758621, 0.9896373,…\n#> $ convex_polar <dbl> 0.000899987, 0.000152532, 0.035230741, 0.00…\n#> $ skinny_polar <dbl> 0.1177954, 0.5440493, 0.7764908, 0.7067394,…\n#> $ stringy_polar <dbl> 1.0000000, 1.0000000, 0.8544140, 1.0000000,…\n#> $ monotonic_polar <dbl> 0.026807610, 0.005554220, 0.068481538, 0.09…\n#> $ outlying_contour <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ skewed_contour <dbl> 0.4954035, 0.5321818, 0.5451171, 0.4675013,…\n#> $ clumpy_contour <dbl> 0.007898114, 0.010846498, 0.020271493, 0.01…\n#> $ sparse_contour <dbl> 0.01449941, 0.01436114, 0.02903878, 0.01451…\n#> $ striated_contour <dbl> 0.9744898, 0.9811321, 0.6766917, 0.9784946,…\n#> $ convex_contour <dbl> 0.000479470, 0.000000000, 0.035908221, 0.00…\n#> $ skinny_contour <dbl> 0.1810093, 1.0000000, 0.7806988, 0.1348225,…\n#> $ stringy_contour <dbl> 1.0000000, 1.0000000, 0.8864787, 1.0000000,…\n#> $ monotonic_contour <dbl> 0.000816260, 0.000008740, 0.001253195, 0.32…\n#> $ num_max_points <dbl> 1, 3, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 6, 1…\n#> $ num_min_points <dbl> 2, 4, 7, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 8, 2…\n#> $ diameter <dbl> 1255.830, 1227.826, 1113.228, 1219.185, 118…\n#> $ area <dbl> 936626.5, 917300.5, 855376.5, 901303.0, 885…\n#> $ perimeter <dbl> 3830.534, 3781.445, 3695.646, 3678.409, 383…\n#> $ physiological_length <dbl> 1253, 1218, 1098, 1205, 1153, 1179, 1188, 1…\n#> $ physiological_width <dbl> 1088, 1091, 1083, 1053, 1078, 1073, 1054, 1…\n#> $ aspect_ratio <dbl> 0.8683160, 0.8957307, 0.9863388, 0.8738589,…\n#> $ rectangularity <dbl> 0.6870470, 0.6903027, 0.7193273, 0.7103222,…\n#> $ circularity <dbl> 0.8021540, 0.8061315, 0.7870211, 0.8370680,…\n#> $ compactness <dbl> 15.66578, 15.58849, 15.96701, 15.01237, 16.…\n#> $ narrow_factor <dbl> 1.154255, 1.125413, 1.027912, 1.157821, 1.1…\n#> $ perimeter_ratio_diameter <dbl> 3.050202, 3.079790, 3.319756, 3.017104, 3.2…\n#> $ perimeter_ratio_length <dbl> 3.520711, 3.466036, 3.412416, 3.493266, 3.5…\n#> $ perimeter_ratio_lw <dbl> 1.636281, 1.637698, 1.694473, 1.629056, 1.7…\n#> $ num_convex_points <dbl> 125, 128, 114, 138, 125, 125, 121, 112, 115…\n#> $ perimeter_convexity <dbl> 0.9404967, 0.9393550, 0.9232549, 0.9502414,…\n#> $ area_convexity <dbl> 0.02359372, 0.03175513, 0.03932713, 0.01740…\n#> $ area_ratio_convexity <dbl> 0.9769501, 0.9692222, 0.9621610, 0.9828892,…\n#> $ equivalent_diameter <dbl> 1092.0393, 1080.7142, 1043.5992, 1071.2491,…\n#> $ eccentricity <dbl> 0.44400376, 0.26866788, 0.31952502, 0.44811…\n#> $ contrast <dbl> 38.69163, 32.73154, 23.20849, 30.14546, 23.…\n#> $ correlation_texture <dbl> 0.9959609, 0.9967740, 0.9975354, 0.9968981,…\n#> $ inverse_difference_moments <dbl> 0.5951133, 0.6022763, 0.6363508, 0.6135506,…\n#> $ entropy <dbl> 6.291552, 6.273267, 5.764995, 6.203471, 5.9…\n#> $ mean_red_val <dbl> 38.02429, 34.55425, 30.92509, 33.44151, 33.…\n#> $ mean_green_val <dbl> 72.38449, 69.89448, 69.22250, 71.32743, 70.…\n#> $ mean_blue_val <dbl> 42.78902, 42.93044, 40.80089, 40.73282, 42.…\n#> $ std_red_val <dbl> 41.66486, 40.48649, 38.55110, 39.09408, 40.…\n#> $ std_green_val <dbl> 74.26665, 73.05275, 76.77596, 75.85529, 75.…\n#> $ std_blue_val <dbl> 46.36232, 48.04195, 48.09177, 46.57170, 48.…\n#> $ correlation <dbl> -0.027629688, -0.003103965, -0.036341630, 0…" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning", - "href": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Leaf data set", + "text": "Leaf data set\n\n\nLakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077." }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning-1", - "href": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning-1", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-2", + "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-2", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Leaf data set", + "text": "Leaf data set\n\n\nLakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077." }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning-2", - "href": "archive/2023-07-nyr/01-introduction.html#what-is-machine-learning-2", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-3", + "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-3", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Leaf data set", + "text": "Leaf data set\n\n\nLakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077." }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#your-turn", - "href": "archive/2023-07-nyr/01-introduction.html#your-turn", - "title": "1 - Introduction", + "objectID": "slides/advanced-04-feature-engineering-part-one.html#your-turn", + "href": "slides/advanced-04-feature-engineering-part-one.html#your-turn", + "title": "4 - Feature engineering: dummies and embeddings", "section": "Your turn", - "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + "text": "Your turn\n\nLoad and explore the leaf data\nMake time to look at the edge column and think about how one would encode it into numerics\n\n\n\n−+\n05:00" }, - { - "objectID": "archive/2023-07-nyr/01-introduction.html#what-is-tidymodels", - "href": "archive/2023-07-nyr/01-introduction.html#what-is-tidymodels", - "title": "1 - Introduction", - "section": "What is tidymodels? ", - "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.1.0 ──\n#> ✔ broom 1.0.5 ✔ rsample 1.1.1.9000\n#> ✔ dials 1.2.0 ✔ tibble 3.2.1 \n#> ✔ dplyr 1.1.2 ✔ tidyr 1.3.0 \n#> ✔ infer 1.0.4 ✔ tune 1.1.1.9001\n#> ✔ modeldata 1.1.0 ✔ workflows 1.1.3 \n#> ✔ parsnip 1.1.0.9003 ✔ workflowsets 1.0.1 \n#> ✔ purrr 1.0.1 ✔ yardstick 1.2.0.9001\n#> ✔ recipes 1.0.6\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Use tidymodels_prefer() to resolve common conflicts." + { + "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-edges", + "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-edges", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Leaf edges", + "text": "Leaf edges\n\n\n\nLow number of unique levels\nSome overlap\n\n\n\nleaf_data |>\n count(edge)\n#> # A tibble: 7 × 2\n#> edge n\n#> <chr> <int>\n#> 1 \"\" 62\n#> 2 \"denate\" 65\n#> 3 \"lobed\" 109\n#> 4 \"lobed, smooth\" 52\n#> 5 \"lobed, toothed\" 109\n#> 6 \"smooth\" 1204\n#> 7 \"toothed\" 306" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game", - "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\nPart of any modelling process is\n\nSplitting your data into training and test set\nUsing a resampling scheme\nFitting models\nAssessing performance\nChoosing a model\nFitting and assessing the final model" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-edges-dummies", + "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-edges-dummies", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Leaf edges dummies", + "text": "Leaf edges dummies\n(technically one-hot encoding)\n\n\n\n\n\n\nX\ndenate\nlobed\nlobed..smooth\nlobed..toothed\nsmooth\ntoothed\n\n\n\n\n0\n0\n0\n0\n1\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n1\n\n\n0\n0\n0\n0\n0\n1\n0\n\n\n0\n0\n0\n0\n0\n1\n0\n\n\n0\n0\n0\n0\n0\n0\n1\n\n\n0\n0\n1\n0\n0\n0\n0" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-1", - "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-1", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#dummy-variables", + "href": "slides/advanced-04-feature-engineering-part-one.html#dummy-variables", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Dummy variables", + "text": "Dummy variables\n\n\nPros\n\nCommonly used\nEasy interpretation\nWill rarely lead to a decrease in performance\n\n\nCons\n\nCan create many columns\nNeeds clean levels\nNot likely the most efficient representation" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-2", - "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-2", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." + "objectID": "slides/advanced-04-feature-engineering-part-one.html#overlapping", + "href": "slides/advanced-04-feature-engineering-part-one.html#overlapping", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Overlapping", + "text": "Overlapping\n\n\nSome of the labels\n\nlobed\nsmooth\ntoothed\nlobed, smooth\nlobed, toothed\n\n\nIf you want to find a “toothed” leaf, which level do you pick?\n\ntoothed\nlobed, toothed\ntoothed and lobed, toothed\n\n\nWe can let lobed, toothed be counted for lobed and toothed" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-3", - "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-3", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Advanced Dummies", + "text": "Advanced Dummies\nThis is basically a poor man’s Natural Language Processing.\n\n\ntokenization -> counting\n\n\nWe think it is a frequent enough case that it is considered its own method.\nWe have 2 variants: “extraction” and “multi choice”" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-4", - "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-4", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction", + "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Advanced Dummies - Extraction", + "text": "Advanced Dummies - Extraction\nWorks on a singular column, using a regular expression to extract the items we want to count.\nDone using regular expressions, either by specifying sep to split the string by, or by using pattern to extract the items.\nAllows for 0 to many items in each string.\nImplemented as step_dummy_extract().\n\nFEAZ" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-5", - "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-5", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Advanced Dummies - Extraction", + "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"\"\n\"denate\"\n\"lobed\"\n\"lobed, smooth\"\n\"lobed, toothed\"\n\"smooth\"\n\"toothed\"\n\n\nUsing\nsep = \", \"\n\nResult\n\n\n\n\n\n\ndenate\nlobed\nsmooth\ntoothed\n\n\n\n\n0\n0\n0\n0\n\n\n1\n0\n0\n0\n\n\n0\n1\n0\n0\n\n\n0\n1\n1\n0\n\n\n0\n1\n0\n1\n\n\n0\n0\n1\n0\n\n\n0\n0\n0\n1" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-6", - "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-6", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-2", + "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-2", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Advanced Dummies - Extraction", + "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"Etching on paper\"\n\"Oil paint on canvas\"\n\"Acrylic paint on paper\"\n\"Oil paint and wax on canvas\"\n\"Oil paint, ink on canvas\"\n\n\nUsing\nsep = \", \"\n\nResult\n\n\n\n\n\n\n\n\n\n\n\n\n\nAcrylic.paint.on.paper\nEtching.on.paper\nink.on.canvas\nOil.paint\nOil.paint.and.wax.on.canvas\nOil.paint.on.canvas\n\n\n\n\n0\n1\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n1\n\n\n1\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n1\n0\n\n\n0\n0\n1\n1\n0\n0" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#the-whole-game-7", - "href": "archive/2023-07-nyr/01-introduction.html#the-whole-game-7", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-3", + "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-3", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Advanced Dummies - Extraction", + "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"Etching on paper\"\n\"Oil paint on canvas\"\n\"Acrylic paint on paper\"\n\"Oil paint and wax on canvas\"\n\"Oil paint, ink on canvas\"\n\n\nUsing\nsep = \"(, )|( and )|( on )\"\n\nResult\n\n\n\n\n\nAcrylic.paint\ncanvas\nEtching\nink\nOil.paint\npaper\nwax\n\n\n\n\n0\n0\n1\n0\n0\n1\n0\n\n\n0\n1\n0\n0\n1\n0\n0\n\n\n1\n0\n0\n0\n0\n1\n0\n\n\n0\n1\n0\n0\n1\n0\n1\n\n\n0\n1\n0\n1\n1\n0\n0" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#lets-install-some-packages", - "href": "archive/2023-07-nyr/01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of RStudio Cloud:\n\ninstall.packages(\"pak\")\n\npkgs <- c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\", \n \"parallelly\", \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \n \"stacks\", \"textrecipes\", \"tidymodels\", \"tidymodels/modeldatatoo\", \n \"vetiver\")\npak::pak(pkgs)\n\n\nCheck Slack (#ml-ws-2023) for an RStudio Cloud link." + "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-4", + "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-4", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Advanced Dummies - Extraction", + "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"['red', 'blue']\"\n\"['red', 'blue', 'white']\"\n\"['blue', 'blue', 'blue']\"\n\n\nUsing\npattern = \"(?<=')[^',]+(?=')\"\n\nResult\n\n\n\n\n\nblue\nred\nwhite\n\n\n\n\n1\n1\n0\n\n\n1\n1\n1\n\n\n3\n0\n0" }, { - "objectID": "archive/2023-07-nyr/01-introduction.html#our-versions", - "href": "archive/2023-07-nyr/01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nbonsai (0.2.1.9000, Github (tidymodels/bonsai@aab79), broom (1.0.5, local), dials (1.2.0, CRAN), doParallel (1.0.17, CRAN), dplyr (1.1.2, CRAN), embed (1.0.0, CRAN), finetune (1.1.0.9000, Github (tidymodels/finetune@52d), ggplot2 (3.4.2, CRAN), lightgbm (3.3.5, CRAN), lme4 (1.1-33, CRAN), modeldata (1.1.0, CRAN), modeldatatoo (0.1.0.9000, Github (tidymodels/modeldatatoo), parallelly (1.36.0, CRAN), parsnip (1.1.0.9003, Github (tidymodels/parsnip@e627), plumber (1.2.1, CRAN), probably (1.0.2, CRAN), purrr (1.0.1, CRAN), ranger (0.15.1, CRAN), recipes (1.0.6, CRAN), rpart (4.1.19, CRAN), rpart.plot (3.1.1, CRAN), rsample (1.1.1.9000, Github (tidymodels/rsample@afc4), scales (1.2.1, CRAN), stacks (1.0.2.9000, local), textrecipes (1.0.2, CRAN), tibble (3.2.1, CRAN), tidymodels (1.1.0, CRAN), tidyr (1.3.0, CRAN), tune (1.1.1.9001, Github (tidymodels/tune@fea8b02), vetiver (0.2.0, CRAN), workflows (1.1.3, CRAN), workflowsets (1.0.1, CRAN), yardstick (1.2.0.9001, Github (tidymodels/yardstick@6c), and Quarto (1.3.433)" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---multi-choice", + "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---multi-choice", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Advanced Dummies - Multi Choice", + "text": "Advanced Dummies - Multi Choice\nWorks on multiple columns, counting items across columns.\nIt can be seen as joining multiple applications of dummy variables together.\nAllows for 0 to many items.\nImplemented as step_dummy_multi_choice().\n\nFEAZ" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglm for generalized linear model (e.g. logistic regression)\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---multi-choice-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---multi-choice-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Advanced Dummies - Multi Choice", + "text": "Advanced Dummies - Multi Choice\n\n\nInput\n\n\n\n\n\nlang_1\nlang_2\nlang_3\n\n\n\n\nEnglish\nNA\nItalian\n\n\nSpanish\nFrench\nFrench\n\n\nNA\nNA\nNA\n\n\nEnglish\nNA\nNA\n\n\n\n\n\n\nResult\n\n\n\n\n\nEnglish\nFrench\nItalian\nSpanish\n\n\n\n\n1\n0\n1\n0\n\n\n0\n1\n0\n1\n\n\n0\n0\n0\n0\n\n\n1\n0\n0\n0" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#othering", + "href": "slides/advanced-04-feature-engineering-part-one.html#othering", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Othering", + "text": "Othering\nBoth step_dummy_extract() and step_dummy_multi_choice() contain a threshold argument.\nThis is used to combine infrequent levels together.\nWe have a step step_other() that does this for nominal variables, but it doesn’t work in these cases, as it has to happen after the extraction/combination." }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-1", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-1", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#othering-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#othering-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Othering", + "text": "Othering\nthreshold = 0 produces 1217 columns.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nX1.monitor\nX1.person\nX1.projection\nX10.light.boxes\nX10.tranformers\nX100.digital.prints\nX100.works\nX11.photographs\nX11.works\nX12.ceramic.tea.mugs\nX12.drawings\nX13.canvases\nX133.slides\nX14.monitors\nX14.photographs\nX14.small.cubes\nX14.works\nX15.engraved.plaques\nX15.hand.coloured.photographs\nX15.photographs\nX15.rooms\nX15.steel.kitchen.ustensils\nX15.unglazed.ceramic.forms\nX151.text.panels\nX16.channels\nX16.mm\nX16.paintings\nX16.sheets\nX160.slides\nX16mm\nX18.drawings\nX18.photographs\nX19.wooden.pieces.of.furniture\nX2.aluminium.panels\nX2.aluminium.tables\nX2.back.projections\nX2.banners\nX2.canvases\nX2.car.batteries\nX2.copper.panels\nX2.digital.prints\nX2.electric.cables\nX2.flat.screens\nX2.fluorescent.tubes\nX2.hooks\nX2.horses\nX2.light.bulbs\nX2.lithographs\nX2.maps\nX2.marker.pens\nX2.metal.scaffold.towers\nX2.monitors\nX2.paintings\nX2.papers\nX2.people\nX2.photographs\nX2.porcelain.light.sockets\nX2.projections\nX2.ropes\nX2.screenprints\nX2.sections.of.pre.cast.concrete.pipe\nX2.sheets\nX2.slippers\nX2.stainless.steel.discs\nX2.trains\nX2.transparencies\nX2.videos\nX2.wall.pieces\nX2.works\nX20.cotton.mattresses\nX20.flat.screens.or.1.projection\nX20.photographs\nX20.wooden.beds\nX2000.gerberas\nX201.photographs\nX21.aluminium.bricks\nX21.channels\nX21.works\nX22.photographs\nX220.works\nX23.monitors\nX24.light.bulbs\nX24.works\nX25.photographs\nX26.works\nX27.photographs\nX28.etchings\nX3.balls\nX3.books\nX3.canvases\nX3.flat.screens\nX3.Kentia.palms.in.3.terracotta.pots\nX3.lamps\nX3.light.bulbs\nX3.monitors\nX3.people\nX3.photographs\nX3.printers\nX3.prints\nX3.projections\nX3.screenprints\nX3.skateboards\nX3.transformers\nX3.wall.mounted.LCD.monitors\nX3.works\nX30.light.emitting.diodes.units\nX30.works\nX31.photographs\nX32.digital.prints\nX32.flat.screens\nX32.papers\nX320.slides\nX34.photographs\nX34.wooden.sculptures\nX35.mm\nX35.works\nX350.digital.prints\nX355.photographs\nX35mm\nX36.tin.dogs\nX39.cardboard.boxes\nX39.metronomes\nX4.aluminium.pixel.boxes.with.DMX.control.box\nX4.Belisha.beacons\nX4.canvases\nX4.channels\nX4.corrugated.slabs.of.concrete\nX4.digital.prints\nX4.digital.prints.with.acrylic.paint\nX4.fibreboard.panels\nX4.flat.screens\nX4.gascookers\nX4.hardboard.walls\nX4.light.bulbs\nX4.mannequins\nX4.monitors\nX4.photocopies\nX4.photographs\nX4.projections\nX4.steel.capsules\nX40.light.emitting.diode.units\nX40.photographs\nX40.works\nX42.electric.lamps\nX42.photographs\nX46.photographs\nX49.Perspex.boxes\nX5.digital.prints\nX5.drawings\nX5.flat.screens\nX5.knives\nX5.monitors\nX5.painted.plaster.busts\nX5.painted.wooden.shelves\nX5.photographs\nX5.projections\nX5.tables\nX5.works\nX50.canvases\nX50.cardboard.boxes\nX50.slides\nX53.photographs\nX54.digital.prints\nX6.digital.prints\nX6.fabrics\nX6.lithographs\nX6.monitors\nX6.Perspex.panels\nX6.photographs\nX6.projections\nX6.works\nX60.digital.prints\nX60.works\nX62.florescent.lights\nX64.tin.soldiers\nX7.channels\nX7.drawings\nX7.light.emitting.diode.columns\nX7.panels\nX7.projections\nX7.stools\nX7.works\nX71.vinyl.records\nX76.works\nX8.channels\nX8.digital.prints\nX8.headphones\nX8.letterpress.prints\nX8.mm\nX8.monitors\nX8.projections\nX8.synthetic.fabric.flags\nX8.woodblock\nX8.works\nX80.slides\nX800.digital.prints\nX84.sleeves\nX9.doors\nX9.LED.lights\nX9.photographs\nX9.works\nX90.photographs\nacetate\nacrylic\nacrylic.fibre\nacrylic.glass\nacrylic.pai\nacrylic.paint\nAcrylic.paint\nacrylic.resin\nacrylic.sheet\nAcrylic.sheet\nacrylic.sheets\nAcrylic.tubes\nadhesive\nadhesive.tape\naerials\nAgfa.Isolette.camera\nAlabaster\nalarm.clocks\nAlkyd.paint\nAltered.Eames.plywood.leg.splints\naluminium\nAluminium\naluminium.buckles\naluminium.disc\naluminium.lightbox\nAluminium.machinery.part\naluminium.paint\naluminium.panel\naluminuim\nAluminuim\naluminum.panel\namplifier\nand.sound\nand.sound..mono\nand.sound..mono.\nand.sound..stereo.\nanimal.bones\nAnimal.intestines\naquatint\nAquatint\naquatint.with.hand.colouring\nartificial.foliage\nartificial.moss\nartificial.sand\nartificial.wig\nartist.s.hair\nash\nAsh\nashtray\naudio\nAudio\naudio.system\naudiotrack\nback.projection\nbadge\nbadges.with.printed.papers\nballasts\nbalsa.wood\nBamboo.cane\nBamboo.poles\nbark\nbaseball.cap\nbath.bombs\nbatteri\nbauxite.rocks\nbeads\nBeanbag\nbeech\nbeer.can\nbells\nbicycle.belt\nbindis\nbins\nblack\nblackboard\nBlind.embossed.print\nboard\nBoard\nBoat\nbone\nbones\nbook\nBook\nbook.cover\nbooklets\nbooks\nboots\nbowl\nbox\nbrass\nBrass\nbrass.pins\nbrass.plate\nbreeze.blocks\nbrick\nBriefcase\nbronze\nBronze\nBronze.bells\nbronze.powder\nBronze.with.silk.scarf\nBronze..with.Jamie.Sargeant\nBronze..with.Nicholas.Sloan\nbrooms\nbuckets\nbuckram\nbuoy\nburlaq.sack\nburnt.wood\nButterflies\nbuttons\ncable\ncables\ncalf\ncalico\ncandle\ncanvas\nCanvas\nCanvas.lining.with.ingrained.dust\ncanvas.paper\nCanvas.tacking.edges\ncarbon.fibre\ncarborundum\ncarborundum.mezzotint\ncard\nCard\ncardboard\nCardboard\ncardboard.architectural.model\ncardboard.base\ncardboard.box\nCardboard.box\ncardboard.boxes\ncardboard.coffin\ncarpet\nCarpet\ncarpets\nCarrara.marble\nCast.iron\ncastor.wheels\nCedar.wood\nceiling\nCellophane\ncellulose\ncellulose.lacquer\ncellulose.print\ncement\ncement.blocks\ncentral.processing.unit\nceramic\nCeramic\nceramic.tiles\nchain\nchair\nchair.parts\nchairs\nchalk\nChalk\ncharcoal\nCharcoal\nchine.collé\nchipboard\nChipboard\nchrome\nchrome.steel.balls\nChromogenic.print\ncibachrome\ncibachrome.print\nCibachrome.print\ncigarette.ash\ncigarette.boxes\ncigarette.butts\ncigarette.sheets\ncigarettes\nclay\nClay\nclothes\ncloves\ncoal\ncoat\nCoca.cola.bottle\ncoconut.oil\ncoffee\ncoin\ncollage\ncollagraph\ncolour\ncolour.and\ncolour.ans.sound..stereo.\ncoloured.graphite\ncoloured.light.bulbs\ncoloured.pencil\ncoloured.pencils\ncolumns\nCommercial.paint\ncompressor\ncompressors\ncomputer\nConcrete\nConté\nContractor.s.shed\nCooked.couscous\ncooker\ncopper\ncopper.sulphate\ncopperplate\ncopperplated\ncoral\ncord\nCork.panels\ncorrection.fluid\nCorrection.fluid\ncotton\nCotton\ncotton.costume\ncotton.thread\ncotton.wool\ncow\nCowhide\ncrayon\nCrayon\ncrude.oil\ncrystalline.particles\ncue.stand\ncues\ncurtains\nCut.paper\nDelabole.slate\ndesks\ndetritus\ndigging.tools\ndigital.image\ndigital.print\nDigital.print\nDigital.print.with.acrylic.paint\ndigital.prints\nDigital.prints.with.acrylic.paint\ndolls\ndoor\ndrawings\nDresden.D1.projector\ndress.shirt\ndry.pigment.paint\ndrypoint\nDrypoint\nduck.tape\ndye\nearth\nEarth\nearthenware\nEarthenware\nelectric.cable\nelectric.wire\nelectrical.a\nelectrical.cable\nelectrical.components\nelectrical.pump\nelectronic.circuit\nelephant.dung\nemulsion\nEmulsion\nenamel\nEnamel\nenamel.button\nenamel.paint\nEnamel.paint\nencre.de.Chine\nEngine\nengraving\nEngraving\nenvelopes\nepoxy\netching\nEtching\nEtching.with.hand.colouring\nexpanding.foam\neyebrow.pencil\neyeshadow\nfabric\nFabric\nfabric.patches\nfaced.particleboard\nfan.heater\nFeathers\nFed.2.type.camera\nfelt\nFelt\nfelt.pen\nFerric.oxide\nfibre.clay\nfibreboard\nFibreboard\nFibreboard.cabinet\nfibreboard.panels\nfibreboard.plinth\nfibreboard.with.clay\nfibreglass\nFibreglass\nfilm\nFilm\nFilm.16.mm\nflat.screen\nflat.screen.or.projector\nflax\nFlies\nFlint\nfloodlight\nflowers\nfluorescent.light\nfluorescent.lights\nfluorescent.site.jacket\nfluorescent.tubes\nflyers\nfoam\nfoam.core\nfoam.rubber\nfoil\nfoil.tape\nfood\nformaldehyde.solution\nformica\nFormica\nfound.posters\nframed.\nFuton.mattress\nGallery.lighting\ngalvanised.steel\ngalvanized.steel\ngel\ngelatin.silver.print\ngelatin.silver.printon.paper\ngelatine.silver.print.with.dye\nglas\nglass\nGlass\nglass.a\nglass.beads\nglass.box\nGlass.chandelier\nglass.vitrine.containing.beauty.and\nGlass.10.hardbacked.books\nglitter\nglove\nglue\ngobo\ngold.leaf\ngold.metallic.paint\nGold.paint\ngouache\nGouache\nGranite\ngraphite\nGraphite\ngrill\nGrpahite\nguitar.strings\ngun\nGunpowder\nhair\nhair.over.metal\nHand.coloured.photograph\nhand.colouring\nHand.cut.book\nhandcolouring\nhardboard\nhardboard.plinth\nhardboard.table\nhardwood\nhat\nheadphones\nHeart.to.Hear\nHeavy.goods.vehicle\nhelium.balloon\nhemp.cord\nhessian\nHessian\nhigh.definition\nhigh.definition_2\nhook\nhorn\nhosepipes\nHousehold.emulsion.paint\nhousehold.gloss.paint\nhousehold.paint\nHousehold.paint\nhuman.hair\nhydraulic.rams\nidentity.card\ninfrared.sensors\ningrained.dust\nink\nInk\ninteractive\nIris.print\niron\niron.powder\njacket\nJacket\njute\nkapok\nkey.\nKilkenny.limestone\nkitchen.utensils\nkite\nknife\nl\nlab.coat\nlacquer\nLacquered.wood\nladder\nlamb\nLambda.print\nlamps\nLamps\nlatex\nlead\nLead\nleather\nLeather\nleather.briefcase\nletterpress\nLetterpress\nlice\nlight\nlight.box\nlight.boxes\nlight.bulb\nlight.bulbs\nlight.control.unit\nlightbox\nLightboxes\nlighter\nlighting.system\nlights\nlinen\nlinocut\nLinocut\nLinocut.print\nlinoleum\nlithograph\nLithograph\nlithograph.with.plasticine\nLithograph.woodcut\nlithographs\nmagnetic.rods\nmagnets\nMahogany\nmango.seeds\nmannequin\nMannequin\nmap\nMap\nmap.pins\nmarble\nMarble\nmarble.base\nmarker.pen\nMarker.pen\nMDF\nMDF.backboard\nme\nmelamine\nmelanine.fibreboard\nMelinex\nmetal\nMetal\nMetal.bicycle\nmetal.bottle.caps\nmetal.bucket\nmetal.chain\nmetal.clamp\nmetal.clothes.rail\nMetal.cot\nmetal.detritus\nmetal.film.spool\nmetal.foil\nMetal.frame\nMetal.frames\nmetal.hook\nmetal.pipe\nmetal.sheet\nmetal.string\nMetal.table\nMetal.tin.cans\nmetal.wire\nmetalcut.prints\nmetallic.powder\nmezzotint\nMezzotint\nmica.flakes\nmicrophone\nmin.DV.cam.tape\nmini.disc.player\nmirror\nMirror\nmirrors\nMiss.Piggy.Bag..and.contents.\nMixed.media\nmobile.pho\nmodelling.putty\nmonitor\nmonitor.or.flat.screen\nmonitor.or.projection\nmono\nMonoprint\nmonotype\nMonotype\nmorse.code.unit\nmotor\nmotorised.base\nmoun\nMountain.bicycle\nMountain.peak.in.container\nmountaineering.equipment\nmounted\nmounted.onto.aluminium\nMud\nmultiple.projections\nmuslin\nMylar.screen\nnails\nnatural.fibres\nneon.gas\nneon.lights\nNeon.lights\nneon.paint\nnewspapers\nnotebook\nnylon\nnylon.straps\nnylon.strings\nNylon.tights\nnyloprints\noak\nOak\noak.table\noak.twig\noffice\noil\noil.paint\nOil.paint\noil.pastel\noil.stick\nOil.stick\noil.tint\non.47.panels\non.aluminium\non.aluminium.panel\non.board\non.cardboard\non.paper\non.paper.between.glass\non.paper.mounted\non.paper.mounted.onto.acrylic.glass\non.paper.mounted.onto.aluminium\non.paper.mounted.onto.aluminium.panel\non.paper.mounted.onto.aluminuim\non.paper.mounted.onto.board\non.paper.mounted.onto.panel\non.paper.mounted.onto.paper\non.paper.mounted.onto.Perspex\non.paper.mounted.onto.plastic\non.paper.with.chalk\non.paper.with.dry.transfert.print.mounted.onto.paper\non.paper.with.paint\non.papers\non.papers.with.paint\non.plastic\non.sticker.paper\non.vinyl.mounted.onto.aluminium\nopen.cell.foam\noptical.gelatin.silver.fibre.print\nor.video\nOrgan\nOstrich.egg\nother.m\nother.materia\nother.materials\nothers.materials\noxidised.brass\np\npaint\nPaint\nPaint.brush\npainted.aluminium\nPainted.aluminium\npainted.boards\nPainted.fibreboard\npainted.glass\npainted.MDF\nPainted.steel\npainted.wall\npainted.wall.text\npainted.wood\nPainted.wood\nPainted.wooden.building.with.asphalt.shingle.roof\npalm.fronds\nPalm.tree\npanel\npaper\nPaper\npaper.mounted\npaper.mounted.onto.aluminium\npaper.mounted.onto.aluminium.panel\npaper.mounted.onto.board\npaper.mounted.onto.foam.core\npaper.mounted.onto.muslin\npaper.mounted.onto.panel\npaper.mounted.onto.paper\npaper.pulp\npaper.tape\npaper.with.dry.transfer.print\npaper.with.dye\npaper.with.ink\npaper.with.oil.paint\npaper.with.watercolour\npaper..Verso..ink\npapers\nparaffin\nparaffin.lamp\npassport\npastel\nPastel\npatina\npeacock.feathers\npen\npendant.lamps\npeople\nPerformance\nperfume.flask.with.pouch\nperspex\nPerspex\nPewter\npharmaceutical.packaging\nphoto.etching\nPhoto.etching\nphotograph\nPhotograph\nphotograph.mounted\nPhotographic.contact.sheet\nphotographs\nPhotographs\nphotographs.\nPiano\npigment\nPigment\nPigment.transfer\npigments\nPine.plywood\npins\nplants\nplaster\nPlaster\nplaster.figures\nplaster.powder\nplasterboard\nplastic\nPlastic\nplastic.bag\nplastic.bags\nplastic.beads\nPlastic.beads\nplastic.box\nplastic.boxes\nplastic.cover\nPlastic.lids\nplastic.pearls\nplastic.pen.lids\nplastic.pillow\nplastic.shoe\nplastic.thread\nplastic.tubes\nplastic.watch.with.photograph\nplastic.watering.can\nplastic.watering.can..spray.bot\nplasticine\nPlasticine\nplatinium.print\nPlexiglas\nplexiglass\nplinth\nplug.boards\nplywood\nPlywood\nplywood.board\npo\nPolychromed.aluminium\nPolyester\npolyester.foam\npolyester.resin\nPolyester.resin\npolyester.textile\npolyfibre\npolymer\npolystyrene\npolystyrene.foam\npolythene\npolyurethane\npolyurethane.foam\nPolyurethane.resin\nPolyurethane.rubber\npolyvinyl.acetate.paint\nPolyvinyl.acetate.paint\nporcelain\nPorcelain\nporcelain.model\nporcelaine\nportable.keyboard.keys\nPortorino.marble\npostcard\nPostcard\npostcards\nposters\nPotato.print\nPowder.coated.aluminium\npowder.coated.steel\nPowder.coated.steel\npowder.paint\npowder.coated.steel_2\nPowder.coated.steel_2\nprint\nPrint\nprinted.map\nprinted.p\nprinted.paper\nPrinted.paper\nprinted.papers\nPrinted.papers\nprojection\nprojection.or.7.monitors\nprojection.or.monitor\npumps\nPVC\nradio\nradio.transmitter\nramin\nrecipes\nRecord.deck\nrecord.player\nrecorded.voice\nreel.to.reel.tape.deck\nrelief\nRelief\nRelief.print\nresin\nResin\nresin.base\nresin.block\nribbon\nRibbon\nRiver.mud\nRoman.vessel\nrope\nropes\nrosary.beads\nrubber\nRubber\nrubber.coated.steel\nRubber.inner.tubes\nrubber.padding\nsafety.pins\nsalt\nsand\nsatin\nsatin.ribbon\nsawdust\nSax.oil.paint\nscanachrome.print\nScanachrome.print\nscissors\nscreenprint\nScreenprint\nScreenprint.with.acrylic.varnish\nScreenprints\nscrews\nsea.shells\nseat\nsection.of.concrete.wall\nsellotape\nSellotape\nSequins\nsewing.machine\nsheep.excrement\nshellac\nshellac.resin\nshells\nShells\nshield\nshirt\nshoeboxes\nshoelace\nshoes\nShop.mannequin\nshown.as\nshown.as.video\nshredder\nsilicon.hose\nsilicon.tubing\nsilicone\nsilicone.adhesive\nsilicone.rubber\nSilicone.rubber\nsilk.tie\nsilkcreen.print\nsilkscreen\nSilkscreen\nSilkscreen.print\nsilver\nsilver.foil\nsilver.gelatin.print\nsilver.solder\nSilver.teapot\nSilverpoint\nsisal.rope\nsisal.string\nSlate\nslide\nSlide\nslides\nsmall.speakers\nsmoke\nsmoke.machine\nso\nsoap\nSoap\nsocks\nSofa\nsoftware\nSoftware\nSondor.playback.machine\nsound\nsound..mono\nsound..mono.\nsound..stereo.\nsound..surround.\nsound.recording\nsound..wood\nspeakers\nspices\nspinning.top\nspray.enamel\nspray.paint\nSpray.paint\nSprayed.Q.Cell\nsreenprint\nStack.of.printed.paper\nstainless.steel\nStainless.steel\nstainless.steel.ashtray\nstainless.steel.screws\nstainless.steel.tea.urn\nstainless.steel.teapot\nstainless.steel.wire\nstand\nstaple.gun\nStarfish\nsteel\nSteel\nsteel.bars\nSteel.bedsprings\nsteel.bracket\nsteel.brake.wires\nsteel.cable\nSteel.locker\nSteel.plate\nsteel.screws\nsteel.shelving\nsteel.wire\nstencil\nstereo\nstereo.\nstickers\nstockings\nstone.base\nstone..with.Jamie.Sargeant\nstool\nstoryboards\nstraw\nstrin\nstring\nstyrene\nstyrofoam\nsugar.paper\nsuitcase\nSuper.16.mm\nSuper.8\nSuper.8.mm\nsweet.wrappers\nswing.seat\nsynthetic.fibre\nsynthetic.fibres\ntable\nTable\ntables\ntape\ntar\ntarpaulin\ntaxidermy\ntea\ntelephone\nTempera\ntent\ntextile\nTextile\ntextiles\nTextiles\ntheatre.light\nthread\nthreads\ntickets\ntights\nTimber\ntissue\ntoilet\ntooth.b\ntorchlight\ntoy.eyeballs\ntracing.paper\ntracking.system\ntram.window\ntransfer.lettering\ntransparency\nTransparency\ntransparent.paper\ntree.branches\ntripod\ntrousers\ntwigs\ntwo.pack.car.lacquer\ntype\ntypescript\ntypewritten.ink\nUHF.Radio\numbrella\nunderpants\nvarnish\nvelvet\nvelvet.p\nVHS.tape\nvideo\nVideo\nvideo.camera\nviewfinder\nvinyl\nVinyl.dispersion\nvinyl.floor.covering\nvinyl.paint\nVinyl.record\nvinyl.seat\nVinyl.tape\nvinyl.text\nvinyl.wall.text\nVinyl.wall.text\nvinyl.wall.texts\nwalkman\nWalkman\nwall\nwall.clock\nWall.painting\nwall.text\nWallpaper\nWashers\nwater\nwatercolour\nWatercolour\nwax\nweb.search.program\nwebbing\nWestern.red.cedar\nwheels\nwhite\nwire\nwire.coat.hanger.\nwires\nwood\nWood\nWood.engraving\nwood.tray\nwood.varnish\nwood.veneer\nWoodblock\nwoodcut\nWoodcut\nWooden.abacus.beads\nwooden.base\nwooden.beams\nWooden.billiard.table\nWooden.birdhouse.with.metal.roof\nwooden.boxes\nWooden.cabinet\nwooden.ceiling.props\nwooden.chair\nWooden.chair\nWooden.construction\nWooden.desk\nwooden.dowels\nwooden.floor\nwooden.frames\nwooden.glass.negative.plate.carrier\nwooden.pallets\nwooden.panel\nwooden.planks\nwooden.platform\nwooden.steps\nwooden.table\nWooden.table\nwooden.trestle\nwool\nWool\nwork.overalls\nworks\nWorry.beads\nwraps\nzinc\nother\n\n\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-2", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-2", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#othering-2", + "href": "slides/advanced-04-feature-engineering-part-one.html#othering-2", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Othering", + "text": "Othering\nthreshold = 0.05 produces 11 columns.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\naquatint\ncanvas\ncolour\nEtching\ngelatin.silver.print\nLithograph\non.paper\npaper\nPhotograph\nScreenprint\nother\n\n\n\n\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n3\n\n\n0\n0\n0\n1\n0\n0\n0\n1\n0\n0\n0\n\n\n0\n0\n0\n1\n0\n0\n0\n1\n0\n0\n0\n\n\n0\n0\n0\n1\n0\n0\n0\n1\n0\n0\n0\n\n\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n1\n\n\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n1" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-3", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-3", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#your-turn-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#your-turn-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Your turn", + "text": "Your turn\n\nFigure out whether step_dummy_extract() or step_dummy_multi_choice() is most appropriate on the edge variable in leaf_data and apply it\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-4", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-4", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#dimensionality-reduction-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#dimensionality-reduction-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Dimensionality Reduction", + "text": "Dimensionality Reduction\nWhat is it?\nTechniques that remove or alter features in order to have fewer but more informative features.\nWhy do we do it?\n\nredundant information\nineffective representation\ncomputational speed" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-5", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-5", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#redundant-information", + "href": "slides/advanced-04-feature-engineering-part-one.html#redundant-information", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Redundant Information", + "text": "Redundant Information\nA feature with no information could be included.\n\nleaf_data |>\n count(outlying_contour)\n#> # A tibble: 1 × 2\n#> outlying_contour n\n#> <dbl> <int>\n#> 1 0 1907" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-6", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-6", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#redundant-information-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#redundant-information-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Redundant Information", + "text": "Redundant Information" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-7", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-7", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation", + "href": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Ineffective Representation", + "text": "Ineffective Representation\nWhile keeping our models in mind, we want to make sure the data is well-suited\n\nCorrelated data\n\nhard for some models\n\nlat/lon compared to distance/angle\n\nhard for most models" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-8", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#to-specify-a-model-8", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Ineffective Representation", + "text": "Ineffective Representation" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-1", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-1", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a different model.\n\n\n\n−+\n05:00\n\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation-2", + "href": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation-2", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Ineffective Representation", + "text": "Ineffective Representation" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#models-well-be-using-today", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#models-well-be-using-today", - "title": "3 - What makes a model?", - "section": "Models we’ll be using today", - "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#computational-speed", + "href": "slides/advanced-04-feature-engineering-part-one.html#computational-speed", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Computational Speed", + "text": "Computational Speed\nDepending on what method we are using and how the data is affected by it, we could see a large reduction in features. This, in turn, leads to a smaller model that is faster to train on.\nOnly exploration and trial and error can determine whether you should use dimensionality reduction techniques. Knowing which methods do what helps you determine what to try." }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#dimensionality-reduction-method", + "href": "slides/advanced-04-feature-engineering-part-one.html#dimensionality-reduction-method", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Dimensionality Reduction Method", + "text": "Dimensionality Reduction Method\n\nZero Variance removal\nPCA\n\nTruncated PCA\nSparse PCA\n\nNNMF\nUMAP\nIsomap" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression-1", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression-1", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#restrictions", + "href": "slides/advanced-04-feature-engineering-part-one.html#restrictions", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Restrictions", + "text": "Restrictions\nAll the methods shown today will not be able to handle\n\nmissing data\nNon-numeric data" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression-2", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#logistic-regression-2", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{distance}\\)\n\nFind a sigmoid line that separates the two classes" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#why-not-t-sne", + "href": "slides/advanced-04-feature-engineering-part-one.html#why-not-t-sne", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Why not t-SNE?", + "text": "Why not t-SNE?\nOne of the main requirements for a feature engineering method is that you can reapply the trained transformation done on the training data set to the testing data set.\nThis is not possible with t-SNE as it is an iterative method that shifts observations in the lower-dimensional space based on their distances to points in the higher-dimensional space.\nIt doesn’t create a mapping that can be reused.\n\nVisualizing Data using t-SNE, Laurens van der Maaten and Geoffrey Hinton, Journal of Machine Learning Research, 2008." }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca", + "href": "slides/advanced-04-feature-engineering-part-one.html#pca", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "PCA", + "text": "PCA\nPrincipal Compoment Analysis is a linear combination of the original data such that most of the variation is captured in the first variable, then the second, then the third, and so on.\n\nFEAZ, FES" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees-1", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees-1", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm", + "href": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "PCA Algorithm", + "text": "PCA Algorithm\nThe first principal component of a set of features \\(X_1, X_2, ..., X_p\\) is the normalized linear combination of the features.\n\\[\nZ_1 = \\phi_{11} X_1 + \\phi_{21} X_2 + ... + \\phi_{p1}X_p\n\\]\nthat has the largest variance under the constraint that \\(\\sum_{j=1}^p \\phi_{j1}^2 = 1\\).\nwe refer to \\(\\phi_{11}, ..., \\phi_{p1}\\) as the loadings of the first principal component.\nAnd think of them as the loading vector \\(\\phi_1\\)." }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees-2", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#decision-trees-2", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "PCA Algorithm", + "text": "PCA Algorithm\nsince we have \\(z_{i1} = \\phi_{11} x_{i1} + \\phi_{21} x_{i2} + ... + \\phi_{p1}x_{ip}\\), then we can write\n\\[\n\\underset{\\phi_{11}, ..., \\phi_{p1}}{\\text{maximize}} \\left\\{ \\dfrac{1}{n} \\sum^n_{i=j} z_{i1} ^2 \\right\\} \\quad \\text{subject to} \\quad \\sum_{j=1}^p \\phi_{j1}^2 = 1\n\\]\nWe are, in essence, maximizing the sample variance of the \\(n\\) values of \\(z_{i1}\\).\nWe refer to \\(z_{11}, ..., z_{n1}\\) as the scores of the first principal component." }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "title": "3 - What makes a model?", - "section": "All models are wrong, but some are useful!", - "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-2", + "href": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-2", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "PCA Algorithm", + "text": "PCA Algorithm\nLuckily, this can be solved using techniques from Linear Algebra, more specifically, it can be solved using an eigen decomposition.\nOne of the main strengths of PCA is that you don’t need to use optimization to get the results without approximations." }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "title": "3 - What makes a model?", - "section": "Workflows bind preprocessors and models", - "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-3", + "href": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-3", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "PCA Algorithm", + "text": "PCA Algorithm\nOnce the first principal component is calculated, we can calculate the second principal component.\nWe find the second principal component \\(Z_2\\) as a linear combination of \\(X_1, ..., X_p\\) that has the maximal variance out of the linear combinations that are uncorrelated with \\(Z_1\\)\nthis is the same as saying that \\(\\phi_2\\) should be orthogonal to the direction \\(\\phi_1\\)" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#what-is-wrong-with-this", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#what-is-wrong-with-this", - "title": "3 - What makes a model?", - "section": "What is wrong with this?", - "text": "What is wrong with this?" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#how-is-that-a-dimensionality-reduction-method", + "href": "slides/advanced-04-feature-engineering-part-one.html#how-is-that-a-dimensionality-reduction-method", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "How is that a dimensionality reduction method?", + "text": "How is that a dimensionality reduction method?\nBy itself, it isn’t, as it rotates all the features in the feature space.\nIt becomes a dimensionality reduction method if we only calculate some of the principal components.\nThis is typically done by retaining a specific number of components or as a threshold on the variance explained." }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#why-a-workflow", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#why-a-workflow", - "title": "3 - What makes a model?", - "section": "Why a workflow()? ", - "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering tomorrow!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#different-percent-variance-plots", + "href": "slides/advanced-04-feature-engineering-part-one.html#different-percent-variance-plots", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "4 different percent variance plots", + "text": "4 different percent variance plots" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-1", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-1", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\ntree_spec %>% \n fit(tip ~ ., data = taxi_train) \n#> parsnip model object\n#> \n#> n= 7045 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 7045 2069 yes (0.70631654 0.29368346) \n#> 2) company=Chicago Independents,City Service,Sun Taxi,Taxicab Insurance Agency Llc,other 4328 744 yes (0.82809612 0.17190388) \n#> 4) distance< 4.615 2365 254 yes (0.89260042 0.10739958) *\n#> 5) distance>=4.615 1963 490 yes (0.75038207 0.24961793) \n#> 10) distance>=12.565 1069 81 yes (0.92422825 0.07577175) *\n#> 11) distance< 12.565 894 409 yes (0.54250559 0.45749441) \n#> 22) company=Chicago Independents,Sun Taxi,Taxicab Insurance Agency Llc 278 71 yes (0.74460432 0.25539568) *\n#> 23) company=City Service,other 616 278 no (0.45129870 0.54870130) \n#> 46) distance< 7.205 178 59 yes (0.66853933 0.33146067) *\n#> 47) distance>=7.205 438 159 no (0.36301370 0.63698630) *\n#> 3) company=Flash Cab,Taxi Affiliation Services 2717 1325 yes (0.51232978 0.48767022) \n#> 6) distance< 3.235 1331 391 yes (0.70623591 0.29376409) *\n#> 7) distance>=3.235 1386 452 no (0.32611833 0.67388167) \n#> 14) distance>=12.39 344 90 yes (0.73837209 0.26162791) *\n#> 15) distance< 12.39 1042 198 no (0.19001919 0.80998081) *" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#applying-pca-with-recipes", + "href": "slides/advanced-04-feature-engineering-part-one.html#applying-pca-with-recipes", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Applying PCA with recipes ", + "text": "Applying PCA with recipes \nEither use the num_comp argument.\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca(all_numeric_predictors(), num_comp = 5)\n\n\nor using the threshold argument\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca(all_numeric_predictors(), threshold = 0.8)" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-2", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-2", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\nworkflow() %>%\n add_formula(tip ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = taxi_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 7045 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 7045 2069 yes (0.70631654 0.29368346) \n#> 2) company=Chicago Independents,City Service,Sun Taxi,Taxicab Insurance Agency Llc,other 4328 744 yes (0.82809612 0.17190388) \n#> 4) distance< 4.615 2365 254 yes (0.89260042 0.10739958) *\n#> 5) distance>=4.615 1963 490 yes (0.75038207 0.24961793) \n#> 10) distance>=12.565 1069 81 yes (0.92422825 0.07577175) *\n#> 11) distance< 12.565 894 409 yes (0.54250559 0.45749441) \n#> 22) company=Chicago Independents,Sun Taxi,Taxicab Insurance Agency Llc 278 71 yes (0.74460432 0.25539568) *\n#> 23) company=City Service,other 616 278 no (0.45129870 0.54870130) \n#> 46) distance< 7.205 178 59 yes (0.66853933 0.33146067) *\n#> 47) distance>=7.205 438 159 no (0.36301370 0.63698630) *\n#> 3) company=Flash Cab,Taxi Affiliation Services 2717 1325 yes (0.51232978 0.48767022) \n#> 6) distance< 3.235 1331 391 yes (0.70623591 0.29376409) *\n#> 7) distance>=3.235 1386 452 no (0.32611833 0.67388167) \n#> 14) distance>=12.39 344 90 yes (0.73837209 0.26162791) *\n#> 15) distance< 12.39 1042 198 no (0.19001919 0.80998081) *" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#your-turn-2", + "href": "slides/advanced-04-feature-engineering-part-one.html#your-turn-2", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Your turn", + "text": "Your turn\n\nApply PCA using step_pca() to leaf_data data set.\nExperiment with different values of num_comp and or threshold.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-3", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#a-model-workflow-3", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\nworkflow(tip ~ ., tree_spec) %>% \n fit(data = taxi_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 7045 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 7045 2069 yes (0.70631654 0.29368346) \n#> 2) company=Chicago Independents,City Service,Sun Taxi,Taxicab Insurance Agency Llc,other 4328 744 yes (0.82809612 0.17190388) \n#> 4) distance< 4.615 2365 254 yes (0.89260042 0.10739958) *\n#> 5) distance>=4.615 1963 490 yes (0.75038207 0.24961793) \n#> 10) distance>=12.565 1069 81 yes (0.92422825 0.07577175) *\n#> 11) distance< 12.565 894 409 yes (0.54250559 0.45749441) \n#> 22) company=Chicago Independents,Sun Taxi,Taxicab Insurance Agency Llc 278 71 yes (0.74460432 0.25539568) *\n#> 23) company=City Service,other 616 278 no (0.45129870 0.54870130) \n#> 46) distance< 7.205 178 59 yes (0.66853933 0.33146067) *\n#> 47) distance>=7.205 438 159 no (0.36301370 0.63698630) *\n#> 3) company=Flash Cab,Taxi Affiliation Services 2717 1325 yes (0.51232978 0.48767022) \n#> 6) distance< 3.235 1331 391 yes (0.70623591 0.29376409) *\n#> 7) distance>=3.235 1386 452 no (0.32611833 0.67388167) \n#> 14) distance>=12.39 344 90 yes (0.73837209 0.26162791) *\n#> 15) distance< 12.39 1042 198 no (0.19001919 0.80998081) *" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-pros-and-cons", + "href": "slides/advanced-04-feature-engineering-part-one.html#pca-pros-and-cons", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "PCA Pros and Cons", + "text": "PCA Pros and Cons\n\n\nPros\n\nFast\nReliable\nExact results (up to sign changes)\n\n\nCons\n\nComputational time is linear in the number of columns\nCan be quite hard to interpret" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-2", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-2", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code to make a workflow with your own model of choice.\n\n\n\n−+\n05:00" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#truncated-pca", + "href": "slides/advanced-04-feature-engineering-part-one.html#truncated-pca", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Truncated PCA", + "text": "Truncated PCA\n\nComputational time is linear in the number of columns\n\nBy default, step_pca() calculates all the loading vectors. And then subset them down to what you need.\nInstead, we can use a different implementation that only calculates what you need. This is what we call truncated PCA.\n\nFEAZ" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#predict-with-your-model", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#predict-with-your-model", - "title": "3 - What makes a model?", - "section": "Predict with your model ", - "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(tip ~ ., tree_spec) %>% \n fit(data = taxi_train)" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#truncated-pca-with-recipes", + "href": "slides/advanced-04-feature-engineering-part-one.html#truncated-pca-with-recipes", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Truncated PCA with recipes ", + "text": "Truncated PCA with recipes \nCan only be done using num_comp\n\nlibrary(embed)\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_truncated(all_numeric_predictors(), num_comp = 5)" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-3", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-3", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = taxi_test)\nWhat do you get?\n\n\n\n−+\n03:00" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca", + "href": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Sparse PCA", + "text": "Sparse PCA\n\nCan be quite hard to interpret\n\nEvery component is a linear combination of all predictors.\n\\[\n\\begin{alignat}{4}\nPC1 &= cyl \\cdot -0.021 &&+ disp \\cdot -0.85 &&+ hp \\cdot -0.52\\\\\nPC2 &= cyl \\cdot 0.013 &&+ disp \\cdot -0.52 &&+ hp \\cdot 0.85\\\\\nPC3 &= cyl \\cdot -0.12 &&+ disp \\cdot 0.016 &&+ hp \\cdot 0.081\\\\\nPC4 &= cyl \\cdot -0.22 &&+ disp \\cdot -0.0061 &&+ hp \\cdot 0.033\\\\\nPC5 &= cyl \\cdot 0.73 &&+ disp \\cdot -0.014 &&+ hp \\cdot 0.0016\n\\end{alignat}\n\\]\nIf we could force some of the loadings to be 0, it would reduce things a lot.\n\nFEAZ" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-4", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-4", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = taxi_test)\nWhat do you get?\n\n\n\n−+\n03:00" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes", + "href": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Sparse PCA with recipes ", + "text": "Sparse PCA with recipes \nCan only be done using num_comp.\nThe predictor_prop argument is used to determine how many zeroes in the loadings.\n\nlibrary(embed)\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_sparse(all_numeric_predictors(), num_comp = 5, predictor_prop = 0.8)" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes-1", + "href": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes-1", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Sparse PCA with recipes ", + "text": "Sparse PCA with recipes \npredictor_prop = 0.8\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_sparse(all_numeric_predictors(), num_comp = 4, predictor_prop = 0.8) |>\n prep() |>\n tidy(number = 2) |>\n pivot_wider(names_from = component, values_from = value) |>\n select(-id)\n#> # A tibble: 10 × 5\n#> terms PC1 PC2 PC3 PC4\n#> <chr> <dbl> <dbl> <dbl> <dbl>\n#> 1 cyl -0.503 0 0.0755 0 \n#> 2 disp -0.504 0 0 0.227 \n#> 3 hp -0.377 -0.228 -0.0985 0 \n#> 4 drat 0.267 -0.262 0 0.911 \n#> 5 wt -0.420 0.105 -0.383 0.236 \n#> 6 qsec 0 0.473 -0.425 0.0966 \n#> 7 vs 0.313 0.203 -0.429 -0.0774 \n#> 8 am 0.0687 -0.440 0.191 -0.00802\n#> 9 gear 0 -0.482 -0.237 -0.208 \n#> 10 carb -0.00616 -0.422 -0.617 -0.0654" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model-1", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model-1", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\nroundint = FALSE is only to quiet a warning" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes-2", + "href": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes-2", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Sparse PCA with recipes ", + "text": "Sparse PCA with recipes \npredictor_prop = 0.2\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_sparse(all_numeric_predictors(), num_comp = 4, predictor_prop = 0.2) |>\n prep() |>\n tidy(number = 2) |>\n pivot_wider(names_from = component, values_from = value) |>\n select(-id)\n#> # A tibble: 10 × 5\n#> terms PC1 PC2 PC3 PC4\n#> <chr> <dbl> <dbl> <dbl> <dbl>\n#> 1 cyl 0.712 0 0 0 \n#> 2 disp 0.702 0 0 0 \n#> 3 hp 0 0 0 0 \n#> 4 drat 0 0 0 0.978\n#> 5 wt 0 0 -0.278 0 \n#> 6 qsec 0 0.666 0 0 \n#> 7 vs 0 0 0 0 \n#> 8 am 0 0 0 0.211\n#> 9 gear 0 -0.746 0 0 \n#> 10 carb 0 0 -0.961 0" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model-2", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#understand-your-model-2", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#nnmf", + "href": "slides/advanced-04-feature-engineering-part-one.html#nnmf", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "NNMF", + "text": "NNMF\nNon-Negative Matrix Factorization is conceptually similar to PCA, but it has different objectives.\nPCA aims to generate uncorrelated components that maximize the variances. One component at a time.\nNNMF, on the other hand, simultaneously optimizes all the components under the constraint that all the loadings are non-negative. While the data is also non-negative.\n\nFEAZ FES" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-5", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-5", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nExtract the model engine object from your fitted workflow.\n⚠️ Never predict() with any extracted components!\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#nnmf-restriction", + "href": "slides/advanced-04-feature-engineering-part-one.html#nnmf-restriction", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "NNMF restriction", + "text": "NNMF restriction\nThe data has to be non-negative, i.e., 0 or higher.\nThis might feel like a pretty big restriction. And that is not wrong. But a lot of data sets end up being naturally non-negative.\nOr could at least be turned into non-negative ones with transformations. As long as you don’t scale them below 0.\nIt makes for much easier interpretations as the loadings don’t cancel each other out like they do for PCA." }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#deploying-a-model", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#deploying-a-model", - "title": "3 - What makes a model?", - "section": "Deploying a model ", - "text": "Deploying a model \nHow do you use your new tree_fit model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"taxi\")\nv\n#> \n#> ── taxi ─ <bundled_workflow> model for deployment \n#> A rpart classification modeling workflow using 6 features\n\nLearn more at https://vetiver.rstudio.com" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#nnmf-pros-and-cons", + "href": "slides/advanced-04-feature-engineering-part-one.html#nnmf-pros-and-cons", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "NNMF Pros and Cons", + "text": "NNMF Pros and Cons\n\n\n\nMore interpretable results\nPulls out better structures\n\n\n\nData must be non-negative\nComputationally expensive\nTraining depends on the seed" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#deploy-your-model-1", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#deploy-your-model-1", - "title": "3 - What makes a model?", - "section": "Deploy your model ", - "text": "Deploy your model \nHow do you use your new model tree_fit in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 2 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/vetiver\n#> ├──/ping (GET)\n#> └──/predict (POST)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#nnmf-with-recipes", + "href": "slides/advanced-04-feature-engineering-part-one.html#nnmf-with-recipes", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "NNMF with recipes ", + "text": "NNMF with recipes \n\nset.seed(1234)\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_nnmf_sparse(all_numeric_predictors(), num_comp = 2) |>\n prep() |>\n tidy(number = 1) |>\n pivot_wider(names_from = component, values_from = value) |>\n select(-id)\n#> # A tibble: 10 × 3\n#> terms NNMF1 NNMF2\n#> <chr> <dbl> <dbl>\n#> 1 am 0 0.00631\n#> 2 carb 0.00428 0.0249 \n#> 3 cyl 0.0134 0.0177 \n#> 4 disp 0.639 0 \n#> 5 drat 0.00545 0.0188 \n#> 6 gear 0.00509 0.0240 \n#> 7 hp 0.294 0.840 \n#> 8 qsec 0.0316 0.0606 \n#> 9 vs 0.000293 0.00247\n#> 10 wt 0.00742 0.00519" }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-6", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#your-turn-6", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n−+\n05:00" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap", + "href": "slides/advanced-04-feature-engineering-part-one.html#umap", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "UMAP", + "text": "UMAP\nUniform Manifold Approximation and Projection is another method that takes high-dimensional data and transforms it into a lower-dimensional space.\nRuns relatively fast and is popular in visualizations." }, { - "objectID": "archive/2023-07-nyr/03-what-makes-a-model.html#the-whole-game---status-update", - "href": "archive/2023-07-nyr/03-what-makes-a-model.html#the-whole-game---status-update", - "title": "3 - What makes a model?", - "section": "The whole game - status update", - "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap-algorithm", + "href": "slides/advanced-04-feature-engineering-part-one.html#umap-algorithm", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "UMAP Algorithm", + "text": "UMAP Algorithm\nRough algorithm\n\nUse spectral embedding to embed points in a low-dimensional space\nCalculate similarity scores between points based on the original data set\nRandomly samples a pair of points based on their similarity scores\nFlips a coin to decide which of the pair of points to give to the other one\nRandomly picks a non-neighbor point to move away from\nMoves the selected point towards its neighbor and away from its non-neighbor\nRepeat 3-6" }, { - "objectID": "archive/2023-07-nyr/08-wrapping-up.html#your-turn", - "href": "archive/2023-07-nyr/08-wrapping-up.html#your-turn", - "title": "8 - Wrapping up", - "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap-parameters", + "href": "slides/advanced-04-feature-engineering-part-one.html#umap-parameters", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "UMAP parameters", + "text": "UMAP parameters\n\n\n\nn_neighbors\n\n\nDetermines how many points are considered neighbors. A point counts as its own neighbor. Lower values lead to a local view.\n\n\n\n\nmin_dist\n\n\nDetermines how close points are allowed to be to each other in the low-dimensional space.\n\n\n\n\nmetric\n\n\nHow distances are calculated in the input data: euclidean, manhattan, jaccard, etc." }, { - "objectID": "archive/2023-07-nyr/08-wrapping-up.html#resources-to-keep-learning", - "href": "archive/2023-07-nyr/08-wrapping-up.html#resources-to-keep-learning", - "title": "8 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Twitter and at the tidyverse blog for updates!" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap-hesitancy", + "href": "slides/advanced-04-feature-engineering-part-one.html#umap-hesitancy", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "UMAP hesitancy", + "text": "UMAP hesitancy\nDue to the flexibility of how this method works, it is almost always possible to generate graphs that appear to have insights in them." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---setup", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---setup", - "title": "2 - Tuning Hyperparameters", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#which-was-created-with-random-data", + "href": "slides/advanced-04-feature-engineering-part-one.html#which-was-created-with-random-data", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Which was created with random data?", + "text": "Which was created with random data?" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---data-usage", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---data-usage", - "title": "2 - Tuning Hyperparameters", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#all-of-them-different-n_neighbors", + "href": "slides/advanced-04-feature-engineering-part-one.html#all-of-them-different-n_neighbors", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "All of them! Different n_neighbors", + "text": "All of them! Different n_neighbors" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---feature-engineering", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#previously---feature-engineering", - "title": "2 - Tuning Hyperparameters", - "section": "Previously - Feature engineering ", - "text": "Previously - Feature engineering \n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap-with-recipes", + "href": "slides/advanced-04-feature-engineering-part-one.html#umap-with-recipes", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "umap with recipes ", + "text": "umap with recipes \n\nlibrary(embed)\nset.seed(1234)\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_umap(all_numeric_predictors()) |>\n prep() |>\n bake(NULL)\n#> # A tibble: 32 × 3\n#> mpg UMAP1 UMAP2\n#> <dbl> <dbl> <dbl>\n#> 1 21 -1.18 1.80 \n#> 2 21 -1.50 1.68 \n#> 3 22.8 0.511 2.91 \n#> 4 21.4 0.613 -0.787\n#> 5 18.7 1.18 -2.83 \n#> 6 18.1 0.144 -0.242\n#> 7 14.3 1.16 -3.18 \n#> 8 24.4 0.598 1.95 \n#> 9 22.8 0.265 1.60 \n#> 10 19.2 -1.17 0.841\n#> # ℹ 22 more rows" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-parameters", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#isomap", + "href": "slides/advanced-04-feature-engineering-part-one.html#isomap", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Isomap", + "text": "Isomap\nIsometric mapping is a non-linear dimensionality reduction method.\nThis is another method that uses distances between points to produce graphs of neighboring points. Where this method is different than other methods is that it uses geodesic distances as opposed to straight-line distances.\nThe geodesic distance is the sum of edge weights along the shortest path between two points.\nThe eigenvectors of the deodesic distance metric are then used to represent the new coordinates." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimize-tuning-parameters", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimize-tuning-parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "slides/advanced-04-feature-engineering-part-one.html#isomap-algorithm", + "href": "slides/advanced-04-feature-engineering-part-one.html#isomap-algorithm", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Isomap Algorithm", + "text": "Isomap Algorithm\nA very high-level description of the Isomap algorithm is given below.\n\nFind the neighbors for each point\nConstruct the neighborhood graph, using Euclidean distance as edge length\nCalculate the shortest path between each pair of points\nUse Multidimensional scaling to compute a lower-dimensional embedding" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "title": "2 - Tuning Hyperparameters", - "section": "Tagging parameters for tuning ", - "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" + "objectID": "slides/advanced-04-feature-engineering-part-one.html#isomap-pros-and-cons", + "href": "slides/advanced-04-feature-engineering-part-one.html#isomap-pros-and-cons", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "Isomap Pros and Cons", + "text": "Isomap Pros and Cons\n\n\nPros\n\nCaptures non-linear effects\nCaptures long-range structure, not just local structure\nNo parameters to set other than neighbors\n\n\nCons\n\nComputationally expensive\nAssumes a single connected manifold" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimizing-the-hash-features", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimizing-the-hash-features", - "title": "2 - Tuning Hyperparameters", - "section": "Optimizing the hash features ", - "text": "Optimizing the hash features \nOur new recipe is:\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\n\nWe will be using a tree-based model in a minute.\n\nThe other categorical predictors are left as-is.\nThat’s why there is no step_dummy()." + "objectID": "slides/advanced-04-feature-engineering-part-one.html#isomap-with-recipes", + "href": "slides/advanced-04-feature-engineering-part-one.html#isomap-with-recipes", + "title": "4 - Feature engineering: dummies and embeddings", + "section": "isomap with recipes ", + "text": "isomap with recipes \n\nlibrary(embed)\nset.seed(1234)\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_isomap(all_numeric_predictors(), neighbors = 10) |>\n prep() |>\n bake(NULL)\n#> # A tibble: 32 × 6\n#> mpg Isomap1 Isomap2 Isomap3 Isomap4 Isomap5\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 21 -98.5 35.6 35.3 137. 105. \n#> 2 21 -98.5 35.6 35.1 137. 104. \n#> 3 22.8 -161. 48.1 20.4 160. 129. \n#> 4 21.4 8.55 14.6 37.2 86.8 125. \n#> 5 18.7 136. 28.5 35.7 -30.8 102. \n#> 6 18.1 -27.7 21.7 38.3 104. 116. \n#> 7 14.3 207. -16.5 34.6 -55.5 158. \n#> 8 24.4 -147. 44.2 23.7 182. 88.4\n#> 9 22.8 -125. 41.1 18.1 151. 118. \n#> 10 19.2 -91.6 33.7 48.6 135. 101. \n#> # ℹ 22 more rows" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-trees", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-trees", - "title": "2 - Tuning Hyperparameters", - "section": "Boosted Trees", - "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved and new samples are predicted using a weighted average of the votes of each tree in the ensemble.\n\nWe’ll focus on the popular lightgbm implementation." + "objectID": "slides/advanced-06-postprocessing.html#startup", + "href": "slides/advanced-06-postprocessing.html#startup", + "title": "6 - Postprocessing", + "section": "Startup! ", + "text": "Startup! \n\nlibrary(tidymodels)\nlibrary(desirability2)\nlibrary(probably)\nlibrary(mirai)\n\n# check torch:\nif (torch::torch_is_installed()) {\n library(torch)\n}\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)\ndaemons(parallel::detectCores())" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands)\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))" + "objectID": "slides/advanced-06-postprocessing.html#more-startup", + "href": "slides/advanced-06-postprocessing.html#more-startup", + "title": "6 - Postprocessing", + "section": "More startup! ", + "text": "More startup! \n\n# Load our example data for this section\n\"https://raw.githubusercontent.com/tidymodels/\" |> \n paste0(\"workshops/main/slides/class_data.RData\") |> \n url() |> \n load()\n\nset.seed(429)\nsim_split <- initial_split(class_data, prop = 0.75, strata = class)\nsim_train <- training(sim_split)\nsim_test <- testing(sim_split)\n\nset.seed(523)\nsim_rs <- vfold_cv(sim_train, v = 10, strata = class)" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "title": "2 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nTBH it is usually not difficult to optimize these models.\n\nOften, there are multiple candidate tuning parameter combinations that have very good results.\n\nTo demonstrate simple concepts, we’ll look at optimizing the number of trees in the ensemble (between 1 and 100) and the learning rate (\\(10^{-5}\\) to \\(10^{-1}\\))." + "objectID": "slides/advanced-06-postprocessing.html#our-neural-network-model", + "href": "slides/advanced-06-postprocessing.html#our-neural-network-model", + "title": "6 - Postprocessing", + "section": "Our neural network model ", + "text": "Our neural network model \n\nrec <- \n recipe(class ~ ., data = sim_train) |> \n step_normalize(all_numeric_predictors())\n\nnnet_spec <- \n mlp(hidden_units = tune(), penalty = tune(), learn_rate = tune(), \n epochs = 100, activation = tune()) |> \n # Remove the class_weights argument\n set_engine(\"brulee\", stop_iter = 10) |> \n set_mode(\"classification\")\n \nnnet_wflow <- workflow(rec, nnet_spec)\n\nnnet_param <- \n nnet_wflow |> \n extract_parameter_set_dials()" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "title": "2 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters ", - "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\")\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)" + "objectID": "slides/advanced-06-postprocessing.html#adjusting-model-predictions", + "href": "slides/advanced-06-postprocessing.html#adjusting-model-predictions", + "title": "6 - Postprocessing", + "section": "Adjusting model predictions", + "text": "Adjusting model predictions\nHow can we modify our predictions? Some examples:\n\nFixing calibration issues*.\nLimit the range of predictions.\nAlternative cutoffs for binary data.\nDeclining to predict.\n\n\n* Requires further estimation (and data).\n\nLet’s first consider the easiest case: alternative cutoffs." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "title": "2 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "slides/advanced-06-postprocessing.html#alternative-thresholds", + "href": "slides/advanced-06-postprocessing.html#alternative-thresholds", + "title": "6 - Postprocessing", + "section": "Alternative thresholds", + "text": "Alternative thresholds\nInstead of up-weighting the samples in the minority (via class_weights), we can try to fit the best model and then define what it means to be an “event.”\n\nInstead of using a 50% threshold, we might lower the level of evidence needed to call a prediction an event.\n\nHow do we tune the threshold?" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search", - "title": "2 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" + "objectID": "slides/advanced-06-postprocessing.html#tailors", + "href": "slides/advanced-06-postprocessing.html#tailors", + "title": "6 - Postprocessing", + "section": "Tailors ", + "text": "Tailors \nThe tailor package is similar to recipes but specifies how to adjust predictions.\nA simple example:\n\nthrsh_tlr <-\n tailor() |>\n adjust_probability_threshold(threshold = 1 / 3)\n\nthrsh_tlr\n\n\nLike a recipe, this initial call doesn’t do anything but declare intent.\nUnlike a recipe, it does not need the data (i.e., predictions) at this point.\n\nRelevant prediction columns are selected when fit() is used (next slide)." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-1", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-1", - "title": "2 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nIn reality we would probably sample the space more densely:" + "objectID": "slides/advanced-06-postprocessing.html#manual-use-of-a-tailor", + "href": "slides/advanced-06-postprocessing.html#manual-use-of-a-tailor", + "title": "6 - Postprocessing", + "section": "Manual use of a tailor ", + "text": "Manual use of a tailor \nThere is a fit() method that requires data and the names of the prediction columns:\n\nthree_rows <- \n tribble(\n ~ class, ~ .pred_class, ~.pred_event, ~.pred_nonevent,\n \"event\", \"event\", 0.6, 0.4,\n \"event\", \"nonevent\", 0.4, 0.6, \n \"nonevent\", \"nonevent\", 0.1, 0.9 \n ) |> \n mutate(across(where(is.character), factor))\n\nthrsh_fit <-\n thrsh_tlr |>\n fit(\n three_rows,\n outcome = class,\n estimate = .pred_class,\n .pred_event:.pred_nonevent # No argument name and order matches factor levels\n )" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#iterative-search", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#iterative-search", - "title": "2 - Tuning Hyperparameters", - "section": "Iterative Search", - "text": "Iterative Search\nWe could start with a few points and search the space:" + "objectID": "slides/advanced-06-postprocessing.html#manual-use-of-a-tailor-1", + "href": "slides/advanced-06-postprocessing.html#manual-use-of-a-tailor-1", + "title": "6 - Postprocessing", + "section": "Manual use of a tailor ", + "text": "Manual use of a tailor \npredict() applies the adjustments:\n\nthrsh_fit\n\npredict(thrsh_fit, three_rows)\n#> # A tibble: 3 × 4\n#> class .pred_class .pred_event .pred_nonevent\n#> <fct> <fct> <dbl> <dbl>\n#> 1 event event 0.6 0.4\n#> 2 event event 0.4 0.6\n#> 3 nonevent nonevent 0.1 0.9" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#parameters", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Parameters", - "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" + "objectID": "slides/advanced-06-postprocessing.html#tailors-within-workflows", + "href": "slides/advanced-06-postprocessing.html#tailors-within-workflows", + "title": "6 - Postprocessing", + "section": "tailors within workflows ", + "text": "tailors within workflows \nIn practice, we would add the tailor to a workflow to make it easier to use:\n\n\nnnet_wflow <- workflow(rec, nnet_spec, thrsh_tlr)\n\n\n\nWe don’t have to set the names of the outcome or prediction columns (yet).\nfit() and predict() happen automatically." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-grid", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-grid", - "title": "2 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\nlgbm_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 4 parameters for tuning\n#> \n#> identifier type object\n#> trees trees nparam[+]\n#> learn_rate learn_rate nparam[+]\n#> agent hash num_terms nparam[+]\n#> company hash num_terms nparam[+]\n\n# Individual functions: \ntrees()\n#> # Trees (quantitative)\n#> Range: [1, 2000]\nlearn_rate()\n#> Learning Rate (quantitative)\n#> Transformer: log-10 [1e-100, Inf]\n#> Range (transformed scale): [-10, -1]\n\n\nA parameter set can be updated (e.g. to change the ranges)." + "objectID": "slides/advanced-06-postprocessing.html#current-adjustments", + "href": "slides/advanced-06-postprocessing.html#current-adjustments", + "title": "6 - Postprocessing", + "section": "Current adjustments", + "text": "Current adjustments\n\nadjust_equivocal_zone(): decline to predict.\nadjust_numeric_calibration(): try to readjust predictions to be consistent with numeric predictions.\nadjust_numeric_range(): restrict the range of predictions.\nadjust_predictions_custom(): similar to dplyr::mutate().\nadjust_probability_calibration(): try to readjust predictions to be consistent with probability predictions.\nadjust_probability_threshold(): custom rule for hard-class predictions from probabilities." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-grid-1", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-grid-1", - "title": "2 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\n\n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1629 0.00000440 524 1454\n#> 2 1746 0.0000000751 1009 2865\n#> 3 53 0.0000180 2313 367\n#> 4 442 0.000000445 347 460\n#> 5 1413 0.0000000208 3232 553\n#> 6 1488 0.0000578 3692 639\n#> 7 906 0.000385 602 332\n#> 8 1884 0.00000000101 1127 567\n#> 9 1812 0.0239 961 1183\n#> 10 393 0.000000117 487 1783\n#> # ℹ 15 more rows\n\n\n\n\nA space-filling design tends to perform better than random grids.\nSpace-filling designs are also usually more efficient than regular grids." + "objectID": "slides/advanced-06-postprocessing.html#some-notes", + "href": "slides/advanced-06-postprocessing.html#some-notes", + "title": "6 - Postprocessing", + "section": "Some notes", + "text": "Some notes\n\nAdjustment order matters; tailor will error early if the ordering rules are violated.\nAdjustments that change class probabilities also affect hard class predictions.\nAdjustments happen before performance estimation.\n\nUndoing something like a log transformation is a bad idea here.\n\nWe have more calibration methods in mind." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn", - "title": "2 - Tuning Hyperparameters", + "objectID": "slides/advanced-06-postprocessing.html#your-turn", + "href": "slides/advanced-06-postprocessing.html#your-turn", + "title": "6 - Postprocessing", "section": "Your turn", - "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n−+\n03:00" + "text": "Your turn\n\nDiscuss with those around you what the “ordering rules” could be.\n\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-regular-grid", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#create-a-regular-grid", - "title": "2 - Tuning Hyperparameters", - "section": "Create a regular grid ", - "text": "Create a regular grid \n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 4)\n\ngrid\n#> # A tibble: 256 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.0000000001 256 256\n#> 2 667 0.0000000001 256 256\n#> 3 1333 0.0000000001 256 256\n#> 4 2000 0.0000000001 256 256\n#> 5 1 0.0000001 256 256\n#> 6 667 0.0000001 256 256\n#> 7 1333 0.0000001 256 256\n#> 8 2000 0.0000001 256 256\n#> 9 1 0.0001 256 256\n#> 10 667 0.0001 256 256\n#> # ℹ 246 more rows" + "objectID": "slides/advanced-06-postprocessing.html#more-notes", + "href": "slides/advanced-06-postprocessing.html#more-notes", + "title": "6 - Postprocessing", + "section": "More Notes", + "text": "More Notes\n\nWhen estimation is required, the data considerations become more complex.\nMost arguments can be tuned.\nFor grid search, we use a conditional execution algorithm that avoids redundant retraining of the preprocessor or model." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn-1", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn-1", - "title": "2 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\n\nWhat advantage would a regular grid have?" + "objectID": "slides/advanced-06-postprocessing.html#tuning-the-probability-threshold", + "href": "slides/advanced-06-postprocessing.html#tuning-the-probability-threshold", + "title": "6 - Postprocessing", + "section": "Tuning the probability threshold ", + "text": "Tuning the probability threshold \n\nthrsh_tlr <-\n tailor() |>\n adjust_probability_threshold(threshold = tune())\n\nnnet_thrsh_wflow <- workflow(rec, nnet_spec, thrsh_tlr)\n \nnnet_thrsh_param <- \n nnet_thrsh_wflow |> \n extract_parameter_set_dials() |> \n update(threshold = threshold(c(0.001, 0.5)))" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#update-parameter-ranges", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#update-parameter-ranges", - "title": "2 - Tuning Hyperparameters", - "section": "Update parameter ranges ", - "text": "Update parameter ranges \n\nlgbm_param <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n update(trees = trees(c(1L, 100L)),\n learn_rate = learn_rate(c(-5, -1)))\n\nset.seed(712)\ngrid <- \n lgbm_param %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 75 0.000312 2991 1250\n#> 2 4 0.0000337 899 3088\n#> 3 15 0.0295 520 1578\n#> 4 8 0.0997 1256 3592\n#> 5 80 0.000622 419 258\n#> 6 70 0.000474 2499 1089\n#> 7 35 0.000165 287 2376\n#> 8 64 0.00137 389 359\n#> 9 58 0.0000250 616 881\n#> 10 84 0.0639 2311 2635\n#> # ℹ 15 more rows" + "objectID": "slides/advanced-06-postprocessing.html#tuning-the-probability-threshold-1", + "href": "slides/advanced-06-postprocessing.html#tuning-the-probability-threshold-1", + "title": "6 - Postprocessing", + "section": "Tuning the probability threshold ", + "text": "Tuning the probability threshold \nNearly the same code as before:\n\n\nctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\ncls_mtr <- metric_set(brier_class, roc_auc, sensitivity, specificity)\n\nset.seed(12)\nnnet_thrsh_res <-\n nnet_thrsh_wflow |>\n tune_grid(\n resamples = sim_rs,\n grid = 25,\n param_info = nnet_thrsh_param, \n control = ctrl,\n metrics = cls_mtr\n )" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#the-results", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#the-results", - "title": "2 - Tuning Hyperparameters", - "section": "The results ", - "text": "The results \n\n\ngrid %>% \n ggplot(aes(trees, learn_rate)) +\n geom_point(size = 4) +\n scale_y_log10()\n\n\n\n\n\n\n\n\n\nNote that the learning rates are uniform on the log-10 scale." + "objectID": "slides/advanced-06-postprocessing.html#grid-results", + "href": "slides/advanced-06-postprocessing.html#grid-results", + "title": "6 - Postprocessing", + "section": "Grid results ", + "text": "Grid results \n\nautoplot(nnet_thrsh_res)" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choosing-tuning-parameters", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choosing-tuning-parameters", - "title": "2 - Tuning Hyperparameters", - "section": "Choosing tuning parameters ", - "text": "Choosing tuning parameters \nLet’s take our previous model and tune more parameters:\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\")\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)\n\n# Update the feature hash ranges (log-2 units)\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "slides/advanced-06-postprocessing.html#grid-results-1", + "href": "slides/advanced-06-postprocessing.html#grid-results-1", + "title": "6 - Postprocessing", + "section": "Grid results", + "text": "Grid results\n\ntanh activation is doing much better.\nthreshold should not (and does not) affect the Brier or ROC metrics.\nWe can achieve low Brier scores.\nWe could run another grid with values <2% for a better threshold estimate." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-3", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-3", - "title": "2 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE)\n\nlgbm_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = 25,\n # The options below are not required by default\n param_info = lgbm_param, \n control = ctrl,\n metrics = reg_metrics\n )\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" + "objectID": "slides/advanced-06-postprocessing.html#multimetric-optimization", + "href": "slides/advanced-06-postprocessing.html#multimetric-optimization", + "title": "6 - Postprocessing", + "section": "Multimetric optimization ", + "text": "Multimetric optimization \n\nnnet_thrsh_res |>\n show_best_desirability(\n maximize(sensitivity),\n minimize(brier_class),\n constrain(specificity, low = 0.8, high = 1.0)\n ) |>\n relocate(threshold, sensitivity, specificity, brier_class, .d_overall)\n#> # A tibble: 5 × 14\n#> threshold sensitivity specificity brier_class .d_overall hidden_units penalty\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <int> <dbl>\n#> 1 0.0218 0.935 0.870 0.0449 0.948 36 2.61e- 5\n#> 2 0.250 0.828 0.951 0.0418 0.902 28 2.61e-10\n#> 3 0.209 0.833 0.936 0.0436 0.896 8 1 e-10\n#> 4 0.188 0.833 0.934 0.0445 0.891 18 5.62e- 2\n#> 5 0.0634 0.899 0.890 0.0527 0.884 40 2.15e- 2\n#> # ℹ 7 more variables: activation <chr>, learn_rate <dbl>, .config <chr>,\n#> # roc_auc <dbl>, .d_max_sensitivity <dbl>, .d_min_brier_class <dbl>,\n#> # .d_box_specificity <dbl>" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-4", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-search-4", - "title": "2 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nlgbm_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [3372/377]> Fold01 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,425 × 9]>\n#> 2 <split [3373/376]> Fold02 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 3 <split [3373/376]> Fold03 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 4 <split [3373/376]> Fold04 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 5 <split [3373/376]> Fold05 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 6 <split [3374/375]> Fold06 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,375 × 9]>\n#> 7 <split [3375/374]> Fold07 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,350 × 9]>\n#> 8 <split [3376/373]> Fold08 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>\n#> 9 <split [3376/373]> Fold09 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>\n#> 10 <split [3376/373]> Fold10 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>" + "objectID": "slides/advanced-06-postprocessing.html#calibration", + "href": "slides/advanced-06-postprocessing.html#calibration", + "title": "6 - Postprocessing", + "section": "Calibration ", + "text": "Calibration \n\n\nmore_sens <-\n nnet_thrsh_res |>\n select_best_desirability(\n maximize(sensitivity),\n minimize(brier_class),\n constrain(specificity, low = 0.8, high = 1.0)\n )\n\nnnet_thrsh_res |>\n collect_predictions(\n parameters = more_sens\n ) |>\n cal_plot_windowed(\n truth = class,\n estimate = .pred_event,\n window_size = 0.2,\n step_size = 0.025,\n )" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-results", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#grid-results", - "title": "2 - Tuning Hyperparameters", - "section": "Grid results ", - "text": "Grid results \n\nautoplot(lgbm_res)" + "objectID": "slides/advanced-06-postprocessing.html#thoughts-about-these-results", + "href": "slides/advanced-06-postprocessing.html#thoughts-about-these-results", + "title": "6 - Postprocessing", + "section": "Thoughts about these results", + "text": "Thoughts about these results\nThe calibration issue in the previous plot shows that some very likely non-events will have underestimated probabilities.\n\nThat may not matter if we are very focused on events.\nThresholding does not affect calibration.\nWe might be able to:\n\nFurther tune the neural network to solve the issue and/or\nAdd a calibration postprocessor" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-results", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-results", - "title": "2 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res)\n#> # A tibble: 50 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 298 19 4.15e- 9 222 36 mae standard 53.2 10 0.427 Preprocessor01_Model1\n#> 2 298 19 4.15e- 9 222 36 rsq standard 0.811 10 0.00785 Preprocessor01_Model1\n#> 3 1394 5 5.82e- 6 28 21 mae standard 52.9 10 0.424 Preprocessor02_Model1\n#> 4 1394 5 5.82e- 6 28 21 rsq standard 0.810 10 0.00857 Preprocessor02_Model1\n#> 5 774 12 4.41e- 2 27 95 mae standard 10.5 10 0.175 Preprocessor03_Model1\n#> 6 774 12 4.41e- 2 27 95 rsq standard 0.939 10 0.00381 Preprocessor03_Model1\n#> 7 1342 7 6.84e-10 71 17 mae standard 53.2 10 0.427 Preprocessor04_Model1\n#> 8 1342 7 6.84e-10 71 17 rsq standard 0.810 10 0.00903 Preprocessor04_Model1\n#> 9 669 39 8.62e- 7 141 145 mae standard 53.2 10 0.426 Preprocessor05_Model1\n#> 10 669 39 8.62e- 7 141 145 rsq standard 0.808 10 0.00661 Preprocessor05_Model1\n#> # ℹ 40 more rows" + "objectID": "slides/advanced-06-postprocessing.html#thoughts-about-the-approach", + "href": "slides/advanced-06-postprocessing.html#thoughts-about-the-approach", + "title": "6 - Postprocessing", + "section": "Thoughts about the approach", + "text": "Thoughts about the approach\nLet’s say that we pick a threshold of 2%. Our explanation to the user/stakeholder would be\n\n“As long as the model is at least 2% sure it is an event, we will call it an event”.\n\nIt may be challenging to convince someone that this is the best option.\n\nThat said, this is probably a better approach than cost-sensitive learning." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-results-1", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#tuning-results-1", - "title": "2 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res, summarize = FALSE)\n#> # A tibble: 500 × 10\n#> id trees min_n learn_rate `agent hash` `company hash` .metric .estimator .estimate .config \n#> <chr> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 Fold01 298 19 0.00000000415 222 36 mae standard 51.8 Preprocessor01_Model1\n#> 2 Fold01 298 19 0.00000000415 222 36 rsq standard 0.834 Preprocessor01_Model1\n#> 3 Fold02 298 19 0.00000000415 222 36 mae standard 52.1 Preprocessor01_Model1\n#> 4 Fold02 298 19 0.00000000415 222 36 rsq standard 0.801 Preprocessor01_Model1\n#> 5 Fold03 298 19 0.00000000415 222 36 mae standard 52.2 Preprocessor01_Model1\n#> 6 Fold03 298 19 0.00000000415 222 36 rsq standard 0.784 Preprocessor01_Model1\n#> 7 Fold04 298 19 0.00000000415 222 36 mae standard 51.7 Preprocessor01_Model1\n#> 8 Fold04 298 19 0.00000000415 222 36 rsq standard 0.828 Preprocessor01_Model1\n#> 9 Fold05 298 19 0.00000000415 222 36 mae standard 55.2 Preprocessor01_Model1\n#> 10 Fold05 298 19 0.00000000415 222 36 rsq standard 0.850 Preprocessor01_Model1\n#> # ℹ 490 more rows" + "objectID": "slides/advanced-06-postprocessing.html#data-to-train-the-adjustments", + "href": "slides/advanced-06-postprocessing.html#data-to-train-the-adjustments", + "title": "6 - Postprocessing", + "section": "Data to train the adjustments", + "text": "Data to train the adjustments\nIf an adjustment requires data, where do we get it from?\n\nFitting a calibration model to the training set re-predictions would be bad.\nAlso, we don’t want to touch the validation or test sets.\n\n\n\nWe need another data set." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choose-a-parameter-combination", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choose-a-parameter-combination", - "title": "2 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \n\nshow_best(lgbm_res, metric = \"rsq\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1890 10 0.0159 115 174 rsq standard 0.940 10 0.00369 Preprocessor12_Model1\n#> 2 774 12 0.0441 27 95 rsq standard 0.939 10 0.00381 Preprocessor03_Model1\n#> 3 1638 36 0.0409 15 120 rsq standard 0.938 10 0.00346 Preprocessor16_Model1\n#> 4 963 23 0.00556 157 13 rsq standard 0.930 10 0.00358 Preprocessor06_Model1\n#> 5 590 5 0.00320 85 73 rsq standard 0.905 10 0.00505 Preprocessor24_Model1" + "objectID": "slides/advanced-06-postprocessing.html#data-sources", + "href": "slides/advanced-06-postprocessing.html#data-sources", + "title": "6 - Postprocessing", + "section": "Data sources", + "text": "Data sources\nTwo possibilities:\n\nShave some data off the training set to create a calibration set.\n\nDuring resampling, we can do the same to the analysis set.\nThe “shaving” process emulates the original sampling method.\nThere are fewer data points for training the preprocessor and primary model.\n\nUse a static calibration set outside our training/validation/testing splits.\n\nCurrently, we have implemented the first method." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "title": "2 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nlgbm_best <- select_best(lgbm_res, metric = \"mae\")\nlgbm_best\n#> # A tibble: 1 × 6\n#> trees min_n learn_rate `agent hash` `company hash` .config \n#> <int> <int> <dbl> <int> <int> <chr> \n#> 1 774 12 0.0441 27 95 Preprocessor03_Model1" + "objectID": "slides/advanced-06-postprocessing.html#example-3-fold-cv", + "href": "slides/advanced-06-postprocessing.html#example-3-fold-cv", + "title": "6 - Postprocessing", + "section": "Example: 3-fold CV", + "text": "Example: 3-fold CV" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#checking-calibration", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#checking-calibration", - "title": "2 - Tuning Hyperparameters", - "section": "Checking Calibration ", - "text": "Checking Calibration \n\n\nlibrary(probably)\nlgbm_res %>%\n collect_predictions(\n parameters = lgbm_best\n ) %>%\n cal_plot_regression(\n truth = avg_price_per_room,\n estimate = .pred,\n alpha = 1 / 3\n )" + "objectID": "slides/advanced-06-postprocessing.html#example-3-fold-cv-internal-split", + "href": "slides/advanced-06-postprocessing.html#example-3-fold-cv-internal-split", + "title": "6 - Postprocessing", + "section": "Example: 3-fold CV internal split", + "text": "Example: 3-fold CV internal split" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#running-in-parallel", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#running-in-parallel", - "title": "2 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" + "objectID": "slides/advanced-06-postprocessing.html#breakdown-for-the-class-imbalance-data", + "href": "slides/advanced-06-postprocessing.html#breakdown-for-the-class-imbalance-data", + "title": "6 - Postprocessing", + "section": "Breakdown for the class imbalance data", + "text": "Breakdown for the class imbalance data\n\n\n\n\n\n\nData\nStrategy\nevent\nno_event\n\n\n\n\nOriginal\nAll\n225\n1775\n\n\nTraining\nNo Calibration\n168\n1331\n\n\nAnalysis\nNo Calibration\n151\n1197\n\n\nTraining\nCalibration\n126\n998\n\n\nAnalysis\nCalibration\n135\n1077\n\n\nCalibration\nCalibration\n16\n120" }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#running-in-parallel-1", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#running-in-parallel-1", - "title": "2 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." + "objectID": "slides/advanced-06-postprocessing.html#calibration-1", + "href": "slides/advanced-06-postprocessing.html#calibration-1", + "title": "6 - Postprocessing", + "section": "Calibration", + "text": "Calibration\nCalibration models, in essence, try to predict the true class using the model predictions. Symbolically:\n\nFor regression models: outcome ~ .pred\nFor binary classifiers: class ~ .pred_class_1\nFor multiclass: class ~ .pred_class_1 + .pred_class_2 + ...\n\nEach calibration method works slightly differently.\nFor example, in regression, a (generalized) linear model is fit, and the residuals are added to new predictions." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "title": "2 - Tuning Hyperparameters", - "section": "Early stopping for boosted trees", - "text": "Early stopping for boosted trees\nWe have directly optimized the number of trees as a tuning parameter.\nInstead we could\n\nSet the number of trees to a single large number.\nStop adding trees when performance gets worse.\n\nThis is known as “early stopping” and there is a parameter for that: stop_iter.\nEarly stopping has a potential to decrease the tuning time." + "objectID": "slides/advanced-06-postprocessing.html#calibration-expectations", + "href": "slides/advanced-06-postprocessing.html#calibration-expectations", + "title": "6 - Postprocessing", + "section": "Calibration expectations", + "text": "Calibration expectations\nKeep expectations low. For these methods to work:\n\nThe systematic issue will need to be large, or at least not subtle.\nA large calibration set is needed to work effectively.\n\nThe example in ALM4TD is an illustrative example and details.\nIn many cases, trying a different model or tuning parameters would be better.\n\nYou can tune the calibration method, one of which is no calibration." }, { - "objectID": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn-2", - "href": "archive/2023-07-nyr/advanced-02-tuning-hyperparameters.html#your-turn-2", - "title": "2 - Tuning Hyperparameters", + "objectID": "slides/advanced-06-postprocessing.html#your-turn-1", + "href": "slides/advanced-06-postprocessing.html#your-turn-1", + "title": "6 - Postprocessing", "section": "Your turn", - "text": "Your turn\n\n\nSet trees = 2000 and tune the stop_iter parameter.\n\n\n\n−+\n10:00" + "text": "Your turn\n\nBased on previous results, choose and fix a specific activation type (i.e., no tune()).\nAdd a calibrator to your tailor with a method = tune() value.\nRun another grid search\n\nDoes it help with this data set?\n\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#previously---setup", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#previously---setup", - "title": "4 - Iterative Search", - "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/advanced-08-wrapping-up.html#your-turn", + "href": "slides/advanced-08-wrapping-up.html#your-turn", + "title": "9 - Wrapping up", + "section": "Your turn", + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#previously---data-usage", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#previously---data-usage", - "title": "4 - Iterative Search", - "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)" + "objectID": "slides/advanced-08-wrapping-up.html#resources-to-keep-learning", + "href": "slides/advanced-08-wrapping-up.html#resources-to-keep-learning", + "title": "9 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://tidymodels.aml4td.org/\n\n\n\n\nhttps://smltar.com/\n\n\n\n\nhttps://feaz-book.com/\n\n\n\nFollow us on Bluesky, Mastodon and at the tidyverse blog for updates!" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#previously---boosting-model", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#previously---boosting-model", - "title": "4 - Iterative Search", - "section": "Previously - Boosting Model", - "text": "Previously - Boosting Model\n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\")\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "slides/extras-iterative-search.html#startup", + "href": "slides/extras-iterative-search.html#startup", + "title": "Extras - Iterative search", + "section": "Startup! ", + "text": "Startup! \n\nlibrary(tidymodels)\nlibrary(important)\nlibrary(probably)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)\nmirai::daemons(parallel::detectCores())" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#iterative-search", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#iterative-search", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#more-startup", + "href": "slides/extras-iterative-search.html#more-startup", + "title": "Extras - Iterative search", + "section": "More startup! ", + "text": "More startup! \n\n# Load our example data for this section\n\"https://raw.githubusercontent.com/tidymodels/\" |> \n paste0(\"workshops/main/slides/class_data.RData\") |> \n url() |> \n load()\n\nset.seed(429)\nsim_split <- initial_split(class_data, prop = 0.75, strata = class)\nsim_train <- training(sim_split)\nsim_test <- testing(sim_split)\n\nset.seed(523)\nsim_rs <- vfold_cv(sim_train, v = 10, strata = class)" + }, + { + "objectID": "slides/extras-iterative-search.html#neural-network", + "href": "slides/extras-iterative-search.html#neural-network", + "title": "Extras - Iterative search", + "section": "Neural network ", + "text": "Neural network \nFrom the seventh advanced slide deck:\n\nnnet_spec <-\n mlp(hidden_units = tune(), penalty = tune(), learn_rate = tune(),\n epochs = 100, activation = tune()\n ) |>\n set_engine(\"brulee\", stop_iter = 10) |>\n set_mode(\"classification\")\n\nrec <- \n recipe(class ~ ., data = sim_train) |> \n step_normalize(all_numeric_predictors())\n \nthrsh_tlr <-\n tailor() |>\n adjust_probability_threshold(threshold = tune()) \n \nnnet_wflow <- workflow(rec, nnet_spec, thrsh_tlr)\n\nnnet_param <-\n nnet_wflow |> \n extract_parameter_set_dials() |> \n update(threshold = threshold(c(0.0001, 0.1)))\n \ncls_mtr <- metric_set(brier_class, roc_auc, sensitivity, specificity)" + }, + { + "objectID": "slides/extras-iterative-search.html#iterative-search", + "href": "slides/extras-iterative-search.html#iterative-search", + "title": "Extras - Iterative search", "section": "Iterative Search", - "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our boosted tree.\n\nWe could do something like:\nmae_pred <- lm(mae ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." + "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our neural network.\n\nWe could do something like:\nbrier_model <- lm(brier ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#iterative-search-1", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#iterative-search-1", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#iterative-search-1", + "href": "slides/extras-iterative-search.html#iterative-search-1", + "title": "Extras - Iterative search", "section": "Iterative Search", - "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between MAE and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" + "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between error and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#a-large-grid", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#a-large-grid", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#a-large-grid", + "href": "slides/extras-iterative-search.html#a-large-grid", + "title": "Extras - Iterative search", "section": "A Large Grid", "text": "A Large Grid" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#a-three-point-grid", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#a-three-point-grid", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#a-three-point-grid", + "href": "slides/extras-iterative-search.html#a-three-point-grid", + "title": "Extras - Iterative search", "section": "A Three Point Grid", "text": "A Three Point Grid" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#gaussian-processes-and-optimization", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#gaussian-processes-and-optimization", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#gaussian-processes-and-optimization", + "href": "slides/extras-iterative-search.html#gaussian-processes-and-optimization", + "title": "Extras - Iterative search", "section": "Gaussian Processes and Optimization", "text": "Gaussian Processes and Optimization\nWe can make a “meta-model” with a small set of historical performance results.\nGaussian Processes (GP) models are a good choice to model performance.\n\nIt is a Bayesian model so we are using Bayesian Optimization (BO).\nFor regression, we can assume that our data are multivariate normal.\nWe also define a covariance function for the variance relationship between data points. A common one is:\n\n\\[\\operatorname{cov}(\\boldsymbol{x}_i, \\boldsymbol{x}_j) = \\exp\\left(-\\frac{1}{2}|\\boldsymbol{x}_i - \\boldsymbol{x}_j|^2\\right) + \\sigma^2_{ij}\\]\n\nGPs are good because\n\nthey are flexible regression models (in the sense that splines are flexible).\nwe need to get mean and variance predictions (and they are Bayesian)\ntheir variability is based on spatial distances.\n\nSome people use random forests (with conformal variance estimates) or other methods but GPs are most popular." }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#predicting-candidates", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#predicting-candidates", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#predicting-candidates", + "href": "slides/extras-iterative-search.html#predicting-candidates", + "title": "Extras - Iterative search", "section": "Predicting Candidates", - "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. MAE)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." + "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. Brier, ROC AUC, RMSE, etc.)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#your-turn", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#your-turn", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#your-turn", + "href": "slides/extras-iterative-search.html#your-turn", + "title": "Extras - Iterative search", "section": "Your turn", - "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize MAE.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" + "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize error.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#gp-fit-ribbon-is-mean---1sd", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#gp-fit-ribbon-is-mean---1sd", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#gp-fit-ribbon-is-mean---1sd", + "href": "slides/extras-iterative-search.html#gp-fit-ribbon-is-mean---1sd", + "title": "Extras - Iterative search", "section": "GP Fit (ribbon is mean +/- 1SD)", "text": "GP Fit (ribbon is mean +/- 1SD)" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#choosing-new-candidates", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#choosing-new-candidates", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#choosing-new-candidates", + "href": "slides/extras-iterative-search.html#choosing-new-candidates", + "title": "Extras - Iterative search", "section": "Choosing New Candidates", "text": "Choosing New Candidates\nThis isn’t a very good fit but we can still use it.\nHow can we use the outputs to choose the next point to measure?\n\nAcquisition functions take the predicted mean and variance and use them to balance:\n\nexploration: new candidates should explore new areas.\nexploitation: new candidates must stay near existing values.\n\nExploration focuses on the variance, exploitation is about the mean." }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#acquisition-functions", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#acquisition-functions", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#acquisition-functions", + "href": "slides/extras-iterative-search.html#acquisition-functions", + "title": "Extras - Iterative search", "section": "Acquisition Functions", "text": "Acquisition Functions\nWe’ll use an acquisition function to select a new candidate.\nThe most popular method appears to be expected improvement (EI) above the current best results.\n\nZero at existing data points.\nThe expected improvement is integrated over all possible improvement (“expected” in the probability sense).\n\nWe would probably pick the point with the largest EI as the next point.\n(There are other functions beyond EI.)" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#expected-improvement", + "href": "slides/extras-iterative-search.html#expected-improvement", + "title": "Extras - Iterative search", "section": "Expected Improvement", "text": "Expected Improvement" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#iteration", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#iteration", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#iteration", + "href": "slides/extras-iterative-search.html#iteration", + "title": "Extras - Iterative search", "section": "Iteration", "text": "Iteration\nOnce we pick the candidate point, we measure performance for it (e.g. resampling).\n\nAnother GP is fit, EI is recomputed, and so on.\n\nWe stop when we have completed the allowed number of iterations or if we don’t see any improvement after a pre-set number of attempts." }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#gp-fit-with-four-points", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#gp-fit-with-four-points", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#gp-fit-with-four-points", + "href": "slides/extras-iterative-search.html#gp-fit-with-four-points", + "title": "Extras - Iterative search", "section": "GP Fit with four points", "text": "GP Fit with four points" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement-1", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement-1", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#expected-improvement-1", + "href": "slides/extras-iterative-search.html#expected-improvement-1", + "title": "Extras - Iterative search", "section": "Expected Improvement", - "text": "Expected Improvement" - }, - { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#gp-evolution", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#gp-evolution", - "title": "4 - Iterative Search", - "section": "GP Evolution", - "text": "GP Evolution" - }, - { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement-evolution", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#expected-improvement-evolution", - "title": "4 - Iterative Search", - "section": "Expected Improvement Evolution", - "text": "Expected Improvement Evolution" + "text": "Expected Improvement\n\n\nAML4TD, TMwR" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#bo-in-tidymodels", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#bo-in-tidymodels", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#bo-in-tidymodels", + "href": "slides/extras-iterative-search.html#bo-in-tidymodels", + "title": "Extras - Iterative search", "section": "BO in tidymodels", "text": "BO in tidymodels\nWe’ll use a function called tune_bayes() that has very similar syntax to tune_grid().\n\nIt has an additional initial argument for the initial set of performance estimates and parameter combinations for the GP model." }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#initial-grid-points", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#initial-grid-points", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#initial-grid-points", + "href": "slides/extras-iterative-search.html#initial-grid-points", + "title": "Extras - Iterative search", "section": "Initial grid points", "text": "Initial grid points\ninitial can be the results of another tune_*() function or an integer (in which case tune_grid() is used under to hood to make such an initial set of results).\n\nWe’ll run the optimization more than once, so let’s make an initial grid of results to serve as the substrate for the BO.\nI suggest at least the number of tuning parameters plus two as the initial grid for BO." }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#an-initial-grid", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#an-initial-grid", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#qualitative-parameters", + "href": "slides/extras-iterative-search.html#qualitative-parameters", + "title": "Extras - Iterative search", + "section": "Qualitative parameters", + "text": "Qualitative parameters\n\nWhat about non-numeric tuning parameters such as activation?\nCurrently, tidymodels converts these to dummy indicators and uses those in the GP. This is not unusual but also not great.\n\nOur initial grid should include more points; one for each level of the qualitative parameter.\nIn our case, the activation function is preset to use 5 possible values.\n\nAn upcoming version of tune will use a different R package to fit the GP that uses factor or Gower kernels. This will avoid making indicators and require fewer initial points." + }, + { + "objectID": "slides/extras-iterative-search.html#an-initial-grid", + "href": "slides/extras-iterative-search.html#an-initial-grid", + "title": "Extras - Iterative search", "section": "An Initial Grid", - "text": "An Initial Grid\n\nreg_metrics <- metric_set(mae, rsq)\n\nset.seed(12)\ninit_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = nrow(lgbm_param) + 2,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(init_res, metric = \"mae\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 390 10 0.0139 13 62 mae standard 11.9 10 0.208 Preprocessor1_Model1\n#> 2 718 31 0.00112 72 25 mae standard 29.1 10 0.325 Preprocessor4_Model1\n#> 3 1236 22 0.0000261 11 17 mae standard 51.8 10 0.416 Preprocessor7_Model1\n#> 4 1044 25 0.00000832 34 12 mae standard 52.8 10 0.424 Preprocessor5_Model1\n#> 5 1599 7 0.0000000402 254 179 mae standard 53.2 10 0.427 Preprocessor6_Model1" + "text": "An Initial Grid\n\nset.seed(12)\ninit_res <-\n nnet_wflow |>\n tune_grid(\n resamples = sim_rs,\n grid = nrow(nnet_param) + 6, # for activation values + 1 extra\n param_info = nnet_param,\n metrics = cls_mtr\n )\n\nshow_best(init_res, metric = \"brier_class\", n = 3) |> select(-.metric, -.estimator)\n#> # A tibble: 3 × 9\n#> hidden_units penalty activation learn_rate threshold mean n std_err\n#> <int> <dbl> <chr> <dbl> <dbl> <dbl> <int> <dbl>\n#> 1 30 0.001 log_sigmo… 0.331 0.0101 0.0410 10 0.00235\n#> 2 6 0.0001 elu 0.0251 0.0800 0.0434 10 0.00173\n#> 3 40 0.0000000001 elu 0.0132 0.0600 0.0453 10 0.00358\n#> # ℹ 1 more variable: .config <chr>" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#bo-using-tidymodels", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#bo-using-tidymodels", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#bo-using-tidymodels", + "href": "slides/extras-iterative-search.html#bo-using-tidymodels", + "title": "Extras - Iterative search", "section": "BO using tidymodels", - "text": "BO using tidymodels\n\nset.seed(15)\nlgbm_bayes_res <-\n lgbm_wflow %>%\n tune_bayes(\n resamples = hotel_rs,\n initial = init_res, # <- initial results\n iter = 20,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(lgbm_bayes_res, metric = \"mae\")\n#> # A tibble: 5 × 12\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config .iter\n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> <int>\n#> 1 1665 2 0.0593 12 59 mae standard 10.1 10 0.173 Iter13 13\n#> 2 1179 2 0.0552 161 121 mae standard 10.2 10 0.147 Iter7 7\n#> 3 1609 6 0.0592 186 192 mae standard 10.2 10 0.195 Iter17 17\n#> 4 1352 6 0.0799 217 46 mae standard 10.3 10 0.211 Iter4 4\n#> 5 1647 4 0.0819 12 240 mae standard 10.3 10 0.198 Iter20 20" + "text": "BO using tidymodels\n\nctrl_bo <- control_bayes(verbose_iter = TRUE, no_improve = Inf) \n\nset.seed(125)\nnnet_bayes_res <-\n nnet_wflow |>\n tune_bayes(\n resamples = sim_rs,\n initial = init_res, # <- initial results\n iter = 25,\n control = ctrl_bo,\n param_info = nnet_param,\n metrics = cls_mtr\n )\n#> Optimizing brier_class using the expected improvement\n#> \n#> ── Iteration 1 ───────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=3, penalty=2.36e-06, activation=log_sigmoid, learn_rate=0.616,\n#> threshold=0.09\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.06833 (+/-0.00667)\n#> \n#> ── Iteration 2 ───────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=38, penalty=3.22e-05, activation=log_sigmoid,\n#> learn_rate=0.578, threshold=0.0678\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04191 (+/-0.0022)\n#> \n#> ── Iteration 3 ───────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=23, penalty=1.47e-08, activation=elu, learn_rate=0.348,\n#> threshold=0.0657\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04808 (+/-0.00476)\n#> \n#> ── Iteration 4 ───────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=19, penalty=0.125, activation=elu, learn_rate=0.00122,\n#> threshold=0.0282\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.05124 (+/-0.00418)\n#> \n#> ── Iteration 5 ───────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=25, penalty=1.15e-10, activation=tanh, learn_rate=0.587,\n#> threshold=0.00897\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.05473 (+/-0.00369)\n#> \n#> ── Iteration 6 ───────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=42, penalty=1.64e-05, activation=log_sigmoid,\n#> learn_rate=0.134, threshold=0.0313\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04167 (+/-0.00294)\n#> \n#> ── Iteration 7 ───────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=2, penalty=1.44e-05, activation=elu, learn_rate=0.592,\n#> threshold=0.0935\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.05835 (+/-0.00898)\n#> \n#> ── Iteration 8 ───────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=23, penalty=0.34, activation=elu, learn_rate=0.0307,\n#> threshold=0.0422\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04357 (+/-0.00432)\n#> \n#> ── Iteration 9 ───────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=50, penalty=3.11e-05, activation=elu, learn_rate=0.443,\n#> threshold=0.0791\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04494 (+/-0.00374)\n#> \n#> ── Iteration 10 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=35, penalty=2.68e-07, activation=log_sigmoid, learn_rate=0.3,\n#> threshold=0.0283\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04413 (+/-0.00326)\n#> \n#> ── Iteration 11 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=48, penalty=0.912, activation=log_sigmoid, learn_rate=0.158,\n#> threshold=0.000413\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.0424 (+/-0.00356)\n#> \n#> ── Iteration 12 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=36, penalty=0.474, activation=log_sigmoid, learn_rate=0.32,\n#> threshold=0.0632\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04258 (+/-0.00239)\n#> \n#> ── Iteration 13 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=43, penalty=0.000343, activation=log_sigmoid,\n#> learn_rate=0.419, threshold=0.00878\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04314 (+/-0.00215)\n#> \n#> ── Iteration 14 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=18, penalty=1.69e-09, activation=elu, learn_rate=0.0127,\n#> threshold=0.043\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04498 (+/-0.00399)\n#> \n#> ── Iteration 15 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=27, penalty=8.91e-05, activation=log_sigmoid, learn_rate=0.63,\n#> threshold=0.063\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04534 (+/-0.0035)\n#> \n#> ── Iteration 16 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=49, penalty=1.11e-05, activation=log_sigmoid,\n#> learn_rate=0.0964, threshold=0.097\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04303 (+/-0.00264)\n#> \n#> ── Iteration 17 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=27, penalty=0.000519, activation=elu, learn_rate=0.0288,\n#> threshold=0.0952\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04144 (+/-0.00259)\n#> \n#> ── Iteration 18 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=40, penalty=0.00206, activation=log_sigmoid, learn_rate=0.159,\n#> threshold=0.01\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04244 (+/-0.00267)\n#> \n#> ── Iteration 19 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=48, penalty=9.79e-10, activation=elu, learn_rate=0.0306,\n#> threshold=0.0726\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04707 (+/-0.00327)\n#> \n#> ── Iteration 20 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=46, penalty=4.16e-05, activation=log_sigmoid,\n#> learn_rate=0.00323, threshold=0.0303\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.1023 (+/-0.0122)\n#> \n#> ── Iteration 21 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=27, penalty=0.0042, activation=elu, learn_rate=0.0214,\n#> threshold=0.0673\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04396 (+/-0.00307)\n#> \n#> ── Iteration 22 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=20, penalty=0.101, activation=log_sigmoid, learn_rate=0.0933,\n#> threshold=0.0109\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.05079 (+/-0.00509)\n#> \n#> ── Iteration 23 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=45, penalty=1.9e-10, activation=elu, learn_rate=0.153,\n#> threshold=0.0949\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04858 (+/-0.0047)\n#> \n#> ── Iteration 24 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=46, penalty=4e-10, activation=elu, learn_rate=0.00275,\n#> threshold=0.0375\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.04534 (+/-0.0042)\n#> \n#> ── Iteration 25 ──────────────────────────────────────────────────────\n#> \n#> i Current best: brier_class=0.04097 (@iter 0)\n#> i Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i hidden_units=21, penalty=7.3e-10, activation=elu, learn_rate=0.0281,\n#> threshold=0.0805\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: brier_class=0.0431 (+/-0.00355)" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results", - "title": "4 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\")" + "objectID": "slides/extras-iterative-search.html#best-results", + "href": "slides/extras-iterative-search.html#best-results", + "title": "Extras - Iterative search", + "section": "Best results", + "text": "Best results\n\nshow_best(nnet_bayes_res, metric = \"brier_class\") |> select(-.metric, -.estimator)\n#> # A tibble: 5 × 10\n#> hidden_units penalty activation learn_rate threshold mean n std_err\n#> <int> <dbl> <chr> <dbl> <dbl> <dbl> <int> <dbl>\n#> 1 30 0.001 log_sigmoid 0.331 0.0101 0.0410 10 0.00235\n#> 2 27 0.000519 elu 0.0288 0.0952 0.0414 10 0.00259\n#> 3 42 0.0000164 log_sigmoid 0.134 0.0313 0.0417 10 0.00294\n#> 4 38 0.0000322 log_sigmoid 0.578 0.0678 0.0419 10 0.00220\n#> 5 48 0.912 log_sigmoid 0.158 0.000413 0.0424 10 0.00356\n#> # ℹ 2 more variables: .config <chr>, .iter <int>" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results-1", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results-1", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#plotting-bo-results", + "href": "slides/extras-iterative-search.html#plotting-bo-results", + "title": "Extras - Iterative search", "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"parameters\")" + "text": "Plotting BO Results\n\nautoplot(nnet_bayes_res, metric = \"brier_class\")" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results-2", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#plotting-bo-results-2", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#plotting-bo-results-1", + "href": "slides/extras-iterative-search.html#plotting-bo-results-1", + "title": "Extras - Iterative search", "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\")" + "text": "Plotting BO Results\n\nautoplot(nnet_bayes_res, metric = \"brier_class\", type = \"parameters\")" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#enhance", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#enhance", - "title": "4 - Iterative Search", - "section": "ENHANCE", - "text": "ENHANCE\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\") +\n ylim(c(9.5, 14))" + "objectID": "slides/extras-iterative-search.html#plotting-bo-results-2", + "href": "slides/extras-iterative-search.html#plotting-bo-results-2", + "title": "Extras - Iterative search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(nnet_bayes_res, metric = \"brier_class\", type = \"performance\")" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#your-turn-1", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#your-turn-1", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#your-turn-1", + "href": "slides/extras-iterative-search.html#your-turn-1", + "title": "Extras - Iterative search", "section": "Your turn", - "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\n\n\n\n−+\n10:00" + "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\nBonus points: Before the optimization is done, press <esc> and see what happens.\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#notes", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#notes", - "title": "4 - Iterative Search", + "objectID": "slides/extras-iterative-search.html#notes", + "href": "slides/extras-iterative-search.html#notes", + "title": "Extras - Iterative search", "section": "Notes", "text": "Notes\n\nStopping tune_bayes() will return the current results.\nParallel processing can still be used to more efficiently measure each candidate point.\nThere are a lot of other iterative methods that you can use.\nThe finetune package also has functions for simulated annealing search." }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#finalizing-the-model", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#finalizing-the-model", - "title": "4 - Iterative Search", - "section": "Finalizing the Model", - "text": "Finalizing the Model\nLet’s say that we’ve tried a lot of different models and we like our lightgbm model the most.\nWhat do we do now?\n\nFinalize the workflow by choosing the values for the tuning parameters.\nFit the model on the entire training set.\nVerify performance using the test set.\nDocument and publish the model(?)" + "objectID": "slides/intro-01-introduction.html#venue-information", + "href": "slides/intro-01-introduction.html#venue-information", + "title": "1 - Introduction", + "section": "Venue information", + "text": "Venue information\n\nThere are gender neutral bathrooms located on floor LL2, next to Chicago A\nA meditation/prayer room is located on floor LL2 in Chicago A\nA lactation room is located on floor LL2 in Chicago B" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#locking-down-the-tuning-parameters", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#locking-down-the-tuning-parameters", - "title": "4 - Iterative Search", - "section": "Locking Down the Tuning Parameters", - "text": "Locking Down the Tuning Parameters\nWe can take the results of the Bayesian optimization and accept the best results:\n\nbest_param <- select_best(lgbm_bayes_res, metric = \"mae\")\nfinal_wflow <- \n lgbm_wflow %>% \n finalize_workflow(best_param)\nfinal_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: boost_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 4 Recipe Steps\n#> \n#> • step_YeoJohnson()\n#> • step_dummy_hash()\n#> • step_dummy_hash()\n#> • step_zv()\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Boosted Tree Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1665\n#> min_n = 2\n#> learn_rate = 0.0592557571004946\n#> \n#> Computational engine: lightgbm" + "objectID": "slides/intro-01-introduction.html#workshop-policies", + "href": "slides/intro-01-introduction.html#workshop-policies", + "title": "1 - Introduction", + "section": "Workshop policies", + "text": "Workshop policies\n\nPlease review the posit::conf code of conduct, which applies to all workshops: https://posit.co/code-of-conduct\nCoC site has info on how to report a problem (in person, email, phone)\nPlease do not photograph people wearing red lanyards" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#the-final-fit", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#the-final-fit", - "title": "4 - Iterative Search", - "section": "The Final Fit", - "text": "The Final Fit\nWe can use individual functions:\nfinal_fit <- final_wflow %>% fit(data = hotel_tr)\n\n# then predict() or augment() \n# then compute metrics\n\nRemember that there is also a convenience function to do all of this:\n\nset.seed(3893)\nfinal_res <- final_wflow %>% last_fit(hotel_split, metrics = reg_metrics)\nfinal_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [3749/1251]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [1,251 × 4]> <workflow>" + "objectID": "slides/intro-01-introduction.html#who-are-you", + "href": "slides/intro-01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have some exposure to basic statistical concepts like linear models and residuals\nYou do not need intermediate or expert familiarity with modeling or ML" }, { - "objectID": "archive/2023-07-nyr/advanced-04-iterative.html#test-set-results", - "href": "archive/2023-07-nyr/advanced-04-iterative.html#test-set-results", - "title": "4 - Iterative Search", - "section": "Test Set Results", - "text": "Test Set Results\n\n\n\nfinal_res %>% \n collect_predictions() %>% \n cal_plot_regression(\n truth = avg_price_per_room, \n estimate = .pred, \n alpha = 1 / 4)\n\nTest set performance:\n\nfinal_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 10.5 Preprocessor1_Model1\n#> 2 rsq standard 0.937 Preprocessor1_Model1" + "objectID": "slides/intro-01-introduction.html#who-are-tidymodels", + "href": "slides/intro-01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\n+ our TA today, Kristin Bott!\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#previously---setup", - "href": "archive/2023-07-nyr/extras-effect-encodings.html#previously---setup", - "title": "Extras - Effect Encodings", - "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/intro-01-introduction.html#section-2", + "href": "slides/intro-01-introduction.html#section-2", + "title": "1 - Introduction", + "section": "", + "text": "Introduce yourself to your neighbors 👋" }, { - "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#previously---data-usage", - "href": "archive/2023-07-nyr/extras-effect-encodings.html#previously---data-usage", - "title": "Extras - Effect Encodings", - "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)" + "objectID": "slides/intro-01-introduction.html#getting-the-materials", + "href": "slides/intro-01-introduction.html#getting-the-materials", + "title": "1 - Introduction", + "section": "Getting the materials", + "text": "Getting the materials\n\nIf you are using Posit Cloud:\n Log in to Posit Cloud (free): TODO-ADD-LATER\n\nIf you are working locally:\n# local download\nusethis::use_course(\"tidymodels/workshops\", destdir = \"some_path\")\n\n# or fork via\nusethis::create_from_github(\"tidymodels/workshops\", fork = TRUE)" }, { - "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "href": "archive/2023-07-nyr/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "title": "Extras - Effect Encodings", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor" + "objectID": "slides/intro-01-introduction.html#asking-for-help", + "href": "slides/intro-01-introduction.html#asking-for-help", + "title": "1 - Introduction", + "section": "Asking for help", + "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" }, { - "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#per-agent-statistics", - "href": "archive/2023-07-nyr/extras-effect-encodings.html#per-agent-statistics", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "slides/intro-01-introduction.html#discord", + "href": "slides/intro-01-introduction.html#discord", + "title": "1 - Introduction", + "section": "Discord ", + "text": "Discord \n\npos.it/conf-event-portal (login)\nClick on “Join Discord, the virtual networking platform!”\nBrowse Channels -> #workshop-tidymodels" }, { - "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#what-is-an-effect-encoding", - "href": "archive/2023-07-nyr/extras-effect-encodings.html#what-is-an-effect-encoding", - "title": "Extras - Effect Encodings", - "section": "What is an effect encoding?", - "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <fct> <int>\n#> 1 52.7 cynthia_worsley 1\n#> 2 51.8 carlos_bryant 2\n#> 3 53.8 lance_hitchcock 3\n#> 4 51.8 lance_hitchcock 4\n#> 5 46.8 cynthia_worsley 5\n#> 6 54.7 charles_najera 6\n#> 7 46.8 cynthia_worsley 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <dbl> <int>\n#> 1 52.7 88.5 1\n#> 2 51.8 89.5 2\n#> 3 53.8 79.8 3\n#> 4 51.8 79.8 4\n#> 5 46.8 88.5 5\n#> 6 54.7 109. 6\n#> 7 46.8 88.5 7\n\n\nThe agent column is replaced with an estimate of the ADR." + "objectID": "slides/intro-01-introduction.html#section-3", + "href": "slides/intro-01-introduction.html#section-3", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#per-agent-statistics-again", - "href": "archive/2023-07-nyr/extras-effect-encodings.html#per-agent-statistics-again", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics again", - "text": "Per-agent statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these means use partial pooling.\nPooling borrows strength across agents and shrinks extreme values towards the mean for agents with very few transations\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for agents with fewer reservations by shrinking the estimate to the overall ADR mean" + "objectID": "slides/intro-01-introduction.html#section-4", + "href": "slides/intro-01-introduction.html#section-4", + "title": "1 - Introduction", + "section": "👀", + "text": "👀\n\n\nhttps://workshops.tidymodels.org" }, { - "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#partial-pooling", - "href": "archive/2023-07-nyr/extras-effect-encodings.html#partial-pooling", - "title": "Extras - Effect Encodings", - "section": "Partial pooling", - "text": "Partial pooling" + "objectID": "slides/intro-01-introduction.html#plan-for-this-workshop", + "href": "slides/intro-01-introduction.html#plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Plan for this workshop", + "text": "Plan for this workshop\n\nYour data budget\nWhat makes a model\nEvaluating models\nTuning models" }, { - "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#agent-effects", - "href": "archive/2023-07-nyr/extras-effect-encodings.html#agent-effects", - "title": "Extras - Effect Encodings", - "section": "Agent effects ", - "text": "Agent effects \n\nlibrary(embed)\n\nhotel_effect_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_YeoJohnson(lead_time) %>%\n step_lencode_mixed(agent, company, outcome = vars(avg_price_per_room)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." + "objectID": "slides/intro-01-introduction.html#what-is-machine-learning", + "href": "slides/intro-01-introduction.html#what-is-machine-learning", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" }, { - "objectID": "archive/2023-07-nyr/extras-effect-encodings.html#effect-encoding-results", - "href": "archive/2023-07-nyr/extras-effect-encodings.html#effect-encoding-results", - "title": "Extras - Effect Encodings", - "section": "Effect encoding results ", - "text": "Effect encoding results \n\nhotel_effect_wflow <-\n workflow() %>%\n add_model(linear_reg()) %>% \n update_recipe(hotel_effect_rec)\n\nreg_metrics <- metric_set(mae, rsq)\n\nhotel_effect_res <-\n hotel_effect_wflow %>%\n fit_resamples(hotel_rs, metrics = reg_metrics)\n\ncollect_metrics(hotel_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.8 10 0.236 Preprocessor1_Model1\n#> 2 rsq standard 0.867 10 0.00377 Preprocessor1_Model1\n\nSlightly worse but it can handle new agents (if they occur)." + "objectID": "slides/intro-01-introduction.html#what-is-machine-learning-1", + "href": "slides/intro-01-introduction.html#what-is-machine-learning-1", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2023-07-nyr/index.html", - "href": "archive/2023-07-nyr/index.html", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." + "objectID": "slides/intro-01-introduction.html#what-is-machine-learning-2025-edition", + "href": "slides/intro-01-introduction.html#what-is-machine-learning-2025-edition", + "title": "1 - Introduction", + "section": "What is machine learning? (2025 edition)", + "text": "What is machine learning? (2025 edition)\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/\n\n\nIn the early 2010s, “Artificial intelligence” (AI) was largely synonymous with what we’ll refer to as “machine learning” in this workshop. In the late 2010s and early 2020s, AI usually referred to deep learning methods. Since the release of ChatGPT in late 2022, “AI” has come to also encompass large language models / generative models." }, { - "objectID": "archive/2023-07-nyr/index.html#welcome", - "href": "archive/2023-07-nyr/index.html#welcome", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." + "objectID": "slides/intro-01-introduction.html#what-is-machine-learning-2", + "href": "slides/intro-01-introduction.html#what-is-machine-learning-2", + "title": "1 - Introduction", + "section": "What is machine learning?", + "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2023-07-nyr/index.html#is-this-workshop-for-me", - "href": "archive/2023-07-nyr/index.html#is-this-workshop-for-me", - "title": "Machine learning with tidymodels", - "section": "Is this workshop for me? ", - "text": "Is this workshop for me? \nThis course assumes intermediate R knowledge. This workshop is for you if:\n\nYou can use the magrittr pipe %>% and/or native pipe |>\nYou are familiar with functions from dplyr, tidyr, and ggplot2\nYou can read data into R, transform and reshape data, and make a wide variety of graphs\n\nWe expect participants to have some exposure to basic statistical concepts, but NOT intermediate or expert familiarity with modeling or machine learning." + "objectID": "slides/intro-01-introduction.html#your-turn", + "href": "slides/intro-01-introduction.html#your-turn", + "title": "1 - Introduction", + "section": "Your turn", + "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + }, + { + "objectID": "slides/intro-01-introduction.html#what-is-tidymodels", + "href": "slides/intro-01-introduction.html#what-is-tidymodels", + "title": "1 - Introduction", + "section": "What is tidymodels? ", + "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.4.1 ──\n#> ✔ broom 1.0.9 ✔ rsample 1.3.1\n#> ✔ dials 1.4.2 ✔ tailor 0.1.0\n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1\n#> ✔ infer 1.0.9 ✔ tune 2.0.0\n#> ✔ modeldata 1.5.1 ✔ workflows 1.3.0\n#> ✔ parsnip 1.3.3 ✔ workflowsets 1.1.1\n#> ✔ purrr 1.1.0 ✔ yardstick 1.3.2\n#> ✔ recipes 1.3.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()" + }, + { + "objectID": "slides/intro-01-introduction.html#the-whole-game", + "href": "slides/intro-01-introduction.html#the-whole-game", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\nRoadmap for today\nMinimal version of predictive modeling process\nFeature engineering and tuning as iterative extensions" + }, + { + "objectID": "slides/intro-01-introduction.html#the-whole-game-1", + "href": "slides/intro-01-introduction.html#the-whole-game-1", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" + }, + { + "objectID": "slides/intro-01-introduction.html#the-whole-game-2", + "href": "slides/intro-01-introduction.html#the-whole-game-2", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." + }, + { + "objectID": "slides/intro-01-introduction.html#the-whole-game-3", + "href": "slides/intro-01-introduction.html#the-whole-game-3", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/index.html#preparation", - "href": "archive/2023-07-nyr/index.html#preparation", - "title": "Machine learning with tidymodels", - "section": "Preparation", - "text": "Preparation\nPlease join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2022.02), available at https://www.rstudio.com/download\nThe following R packages, which you can install from the R console:\n\n\n# First, install the pak package: \n\ninstall.packages(\"pak\")\n\n# Then the packages for both days\npkgs <- \n c(\"bonsai\", \"doParallel\", \"finetune\", \"lightgbm\", \"lme4\", \"plumber\", \n \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"stacks\", \"textrecipes\", \n \"tidymodels\", \"tidymodels/modeldatatoo\", \"vetiver\")\npak::pak(pkgs)" + "objectID": "slides/intro-01-introduction.html#the-whole-game-4", + "href": "slides/intro-01-introduction.html#the-whole-game-4", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/index.html#slides", - "href": "archive/2023-07-nyr/index.html#slides", - "title": "Machine learning with tidymodels", - "section": "Slides", - "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. There are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to tidymodels\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n\n\n\nAdvanced tidymodels\n\n01: Feature engineering using recipes\n02: Tuning hyperparameters (grid search)\n03: Grid search via racing\n04: Iterative search\n\n\n\nExtra content (time permitting)\n\nTransit Case Study (includes stacking)\nEffect encodings\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." + "objectID": "slides/intro-01-introduction.html#the-whole-game-5", + "href": "slides/intro-01-introduction.html#the-whole-game-5", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/index.html#code", - "href": "archive/2023-07-nyr/index.html#code", - "title": "Machine learning with tidymodels", - "section": "Code", - "text": "Code\nQuarto files (version 1.4.104) for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" + "objectID": "slides/intro-01-introduction.html#the-whole-game-6", + "href": "slides/intro-01-introduction.html#the-whole-game-6", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/index.html#past-workshops", - "href": "archive/2023-07-nyr/index.html#past-workshops", - "title": "Machine learning with tidymodels", - "section": "Past workshops", - "text": "Past workshops\n\nJuly 2022 at rstudio::conf()\nAugust 2022 in Reykjavik\nJuly 2023 at the New York R Conference" + "objectID": "slides/intro-01-introduction.html#the-whole-game-7", + "href": "slides/intro-01-introduction.html#the-whole-game-7", + "title": "1 - Introduction", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2023-07-nyr/index.html#acknowledgments", - "href": "archive/2023-07-nyr/index.html#acknowledgments", - "title": "Machine learning with tidymodels", - "section": "Acknowledgments", - "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." + "objectID": "slides/intro-01-introduction.html#lets-install-some-packages", + "href": "slides/intro-01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"lightgbm\", \"lme4\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)" }, { - "objectID": "archive/2023-07-nyr/index.html#reuse-and-licensing", - "href": "archive/2023-07-nyr/index.html#reuse-and-licensing", - "title": "Machine learning with tidymodels", - "section": "Reuse and licensing", - "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." + "objectID": "slides/intro-01-introduction.html#our-versions", + "href": "slides/intro-01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nR version 4.5.0 (2025-04-11), Quarto (1.7.31)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.4.0\n\n\nbroom\n1.0.9\n\n\nCubist\n0.5.0\n\n\ndials\n1.4.2\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.4\n\n\nembed\n1.2.0\n\n\nfinetune\n1.2.1\n\n\nforested\n0.2.0\n\n\nFormula\n1.2-5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nggplot2\n3.5.2\n\n\nlattice\n0.22-7\n\n\nlightgbm\n4.6.0\n\n\nlme4\n1.1-37\n\n\nmodeldata\n1.5.1\n\n\nparallelly\n1.45.1\n\n\nparsnip\n1.3.3\n\n\nplotmo\n3.6.4\n\n\nplotrix\n3.8-4\n\n\nplumber\n1.3.0\n\n\nprobably\n1.1.1\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\npurrr\n1.1.0\n\n\nranger\n0.17.0\n\n\nrecipes\n1.3.1\n\n\nrpart\n4.1.24\n\n\nrpart.plot\n3.1.3\n\n\nrsample\n1.3.1\n\n\nrules\n1.0.2\n\n\nscales\n1.4.0\n\n\nsplines2\n0.5.4\n\n\nstacks\n1.1.1\n\n\ntailor\n0.1.0\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntext2vec\n0.6.4\n\n\ntextrecipes\n1.1.0\n\n\ntidymodels\n1.4.1\n\n\ntidyr\n1.3.1\n\n\ntune\n2.0.0\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.3.0\n\n\nworkflowsets\n1.1.1\n\n\nyardstick\n1.3.2" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#working-with-our-predictors", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#working-with-our-predictors", - "title": "1 - Feature Engineering", - "section": "Working with our predictors", - "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for linear regression).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." + "objectID": "slides/intro-03-what-makes-a-model.html#your-turn", + "href": "slides/intro-03-what-makes-a-model.html#your-turn", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets\nbrulee for regression using torch" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#what-is-feature-engineering", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#what-is-feature-engineering", - "title": "1 - Feature Engineering", - "section": "What is feature engineering?", - "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPrincipal component analysis (PCA) feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection (FES)." + "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model", + "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\n\nChoose a model type\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#example-dates", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#example-dates", - "title": "1 - Feature Engineering", - "section": "Example: Dates", - "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." + "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-1", + "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-1", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#general-definitions", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#general-definitions", - "title": "1 - Feature Engineering", - "section": "General definitions ", - "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." + "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-2", + "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-2", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model type\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#previously---setup", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#previously---setup", - "title": "1 - Feature Engineering", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\n\n# Add another package:\nlibrary(textrecipes)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-3", + "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-3", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() |>\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#previously---data-usage", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#previously---data-usage", - "title": "1 - Feature Engineering", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\n\nWe’ll go from here and create a set of resamples to use for model assessments." + "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-4", + "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-4", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\nlogistic_reg() |>\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#resampling-strategy", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#resampling-strategy", - "title": "1 - Feature Engineering", - "section": "Resampling Strategy", - "text": "Resampling Strategy" + "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-5", + "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-5", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model type\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#resampling-strategy-1", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#resampling-strategy-1", - "title": "1 - Feature Engineering", - "section": "Resampling Strategy ", - "text": "Resampling Strategy \nWe’ll use simple 10-fold cross-validation (stratified sampling):\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)\nhotel_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [3372/377]> Fold01\n#> 2 <split [3373/376]> Fold02\n#> 3 <split [3373/376]> Fold03\n#> 4 <split [3373/376]> Fold04\n#> 5 <split [3373/376]> Fold05\n#> 6 <split [3374/375]> Fold06\n#> 7 <split [3375/374]> Fold07\n#> 8 <split [3376/373]> Fold08\n#> 9 <split [3376/373]> Fold09\n#> 10 <split [3376/373]> Fold10" + "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-6", + "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-6", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", - "title": "1 - Feature Engineering", - "section": "Prepare your data for modeling ", - "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of preprocessing and feature engineering steps.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." + "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-7", + "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-7", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree() |> \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#a-first-recipe", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#a-first-recipe", - "title": "1 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" + "objectID": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-8", + "href": "slides/intro-03-what-makes-a-model.html#to-specify-a-model-8", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model type\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#a-first-recipe-1", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#a-first-recipe-1", - "title": "1 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nsummary(hotel_rec)\n#> # A tibble: 27 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 lead_time <chr [2]> predictor original\n#> 2 stays_in_weekend_nights <chr [2]> predictor original\n#> 3 stays_in_week_nights <chr [2]> predictor original\n#> 4 adults <chr [2]> predictor original\n#> 5 children <chr [2]> predictor original\n#> 6 babies <chr [2]> predictor original\n#> 7 meal <chr [3]> predictor original\n#> 8 country <chr [3]> predictor original\n#> 9 market_segment <chr [3]> predictor original\n#> 10 distribution_channel <chr [3]> predictor original\n#> # ℹ 17 more rows\n\nThe type column contains information on the variables" + "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-1", + "href": "slides/intro-03-what-makes-a-model.html#your-turn-1", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a logistic regression model.\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/\n\n\nExtension/Challenge: Edit this code to use a different model. For example, try using a conditional inference tree as implemented in the partykit package by changing the engine - or try an entirely different model type!\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn", - "title": "1 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\nWhat do you think are in the type vectors for the lead_time and country columns?\n\n\n\n−+\n02:00" + "objectID": "slides/intro-03-what-makes-a-model.html#models-well-be-using-today", + "href": "slides/intro-03-what-makes-a-model.html#models-well-be-using-today", + "title": "3 - What makes a model?", + "section": "Models we’ll be using today", + "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#create-indicator-variables", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#create-indicator-variables", - "title": "1 - Feature Engineering", - "section": "Create indicator variables ", - "text": "Create indicator variables \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns.\nstep_dummy() records the levels of the categorical predictors in the training set." + "objectID": "slides/intro-03-what-makes-a-model.html#logistic-regression", + "href": "slides/intro-03-what-makes-a-model.html#logistic-regression", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#filter-out-constant-columns", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#filter-out-constant-columns", - "title": "1 - Feature Engineering", - "section": "Filter out constant columns ", - "text": "Filter out constant columns \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" + "objectID": "slides/intro-03-what-makes-a-model.html#logistic-regression-1", + "href": "slides/intro-03-what-makes-a-model.html#logistic-regression-1", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#normalization", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#normalization", - "title": "1 - Feature Engineering", - "section": "Normalization ", - "text": "Normalization \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." + "objectID": "slides/intro-03-what-makes-a-model.html#logistic-regression-2", + "href": "slides/intro-03-what-makes-a-model.html#logistic-regression-2", + "title": "3 - What makes a model?", + "section": "Logistic regression", + "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nFind a sigmoid line that separates the two classes" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#reduce-correlation", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#reduce-correlation", - "title": "1 - Feature Engineering", - "section": "Reduce correlation ", - "text": "Reduce correlation \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." + "objectID": "slides/intro-03-what-makes-a-model.html#decision-trees", + "href": "slides/intro-03-what-makes-a-model.html#decision-trees", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps", - "title": "1 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" + "objectID": "slides/intro-03-what-makes-a-model.html#decision-trees-1", + "href": "slides/intro-03-what-makes-a-model.html#decision-trees-1", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-1", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-1", - "title": "1 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = vars(avg_price_per_room))\n\n\nA fancy machine learning supervised dimension reduction technique…\n\nNote that this uses the outcome, and it is from an extension package" + "objectID": "slides/intro-03-what-makes-a-model.html#decision-trees-2", + "href": "slides/intro-03-what-makes-a-model.html#decision-trees-2", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-2", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-2", - "title": "1 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" + "objectID": "slides/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "href": "slides/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "title": "3 - What makes a model?", + "section": "All models are wrong, but some are useful!", + "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" + }, + { + "objectID": "slides/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "href": "slides/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "title": "3 - What makes a model?", + "section": "Workflows bind preprocessors and models", + "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn-1", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn-1", - "title": "1 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\n\nCreate a recipe() for the hotel data to:\n\nuse a Yeo-Johnson (YJ) transformation on lead_time\nconvert factors to indicator variables\nremove zero-variance variables\nadd the spline technique shown above\n\n\n\n\n−+\n03:00" + "objectID": "slides/intro-03-what-makes-a-model.html#what-is-wrong-with-this", + "href": "slides/intro-03-what-makes-a-model.html#what-is-wrong-with-this", + "title": "3 - What makes a model?", + "section": "What is wrong with this?", + "text": "What is wrong with this?\n \n\n\nThe source of data leakage here is preprocessing on the whole source data set rather than on the whole training set. So, the risk is “the values from the test set affect the mean and standard deviations used in the preprocessing.” That discussion shouldn’t mention metrics, as they haven’t been introduced yet.\nWhile centering + scaling are easy to explain the general idea with, it’s not necessarily the biggest danger in practice. The more elaborate the estimation in your preprocessing, the higher the risk of overfitting coming back to bite you. An example of a more elaborate preprocessing estimation is principal components." }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#minimal-recipe", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#minimal-recipe", - "title": "1 - Feature Engineering", - "section": "Minimal recipe ", - "text": "Minimal recipe \n\nhotel_indicators <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)" + "objectID": "slides/intro-03-what-makes-a-model.html#why-a-workflow", + "href": "slides/intro-03-what-makes-a-model.html#why-a-workflow", + "title": "3 - What makes a model?", + "section": "Why a workflow()? ", + "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on this in Getting More Out of Feature Engineering and Tuning for Machine Learning)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#measuring-performance", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#measuring-performance", - "title": "1 - Feature Engineering", - "section": "Measuring Performance ", - "text": "Measuring Performance \nWe’ll compute two measures: mean absolute error and the coefficient of determination (a.k.a \\(R^2\\)).\n\\[\\begin{align}\nMAE &= \\frac{1}{n}\\sum_{i=1}^n |y_i - \\hat{y}_i| \\notag \\\\\nR^2 &= cor(y_i, \\hat{y}_i)^2\n\\end{align}\\]\nThe focus will be on MAE for parameter optimization. We’ll use a metric set to compute these:\n\nreg_metrics <- metric_set(mae, rsq)" + "objectID": "slides/intro-03-what-makes-a-model.html#a-model-workflow-1", + "href": "slides/intro-03-what-makes-a-model.html#a-model-workflow-1", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\ntree_spec |> \n fit(forested ~ ., data = forested_train) \n#> parsnip model object\n#> \n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#using-a-workflow", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#using-a-workflow", - "title": "1 - Feature Engineering", - "section": "Using a workflow ", - "text": "Using a workflow \n\nset.seed(9)\n\nhotel_lm_wflow <-\n workflow() %>%\n add_recipe(hotel_indicators) %>%\n add_model(linear_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nhotel_lm_res <-\n hotel_lm_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_lm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.6 10 0.214 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00339 Preprocessor1_Model1" + "objectID": "slides/intro-03-what-makes-a-model.html#a-model-workflow-2", + "href": "slides/intro-03-what-makes-a-model.html#a-model-workflow-2", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\nworkflow() |>\n add_formula(forested ~ .) |>\n add_model(tree_spec) |>\n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn-2", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#your-turn-2", - "title": "1 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n−+\n05:00" + "objectID": "slides/intro-03-what-makes-a-model.html#a-model-workflow-3", + "href": "slides/intro-03-what-makes-a-model.html#a-model-workflow-3", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\nworkflow(forested ~ ., tree_spec) |> \n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#holdout-predictions", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#holdout-predictions", - "title": "1 - Feature Engineering", - "section": "Holdout predictions ", - "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nlm_cv_pred <- collect_predictions(hotel_lm_res)\nlm_cv_pred %>% print(n = 7)\n#> # A tibble: 3,749 × 5\n#> id .pred .row avg_price_per_room .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 Fold01 75.1 20 40 Preprocessor1_Model1\n#> 2 Fold01 49.3 28 54 Preprocessor1_Model1\n#> 3 Fold01 64.9 45 50 Preprocessor1_Model1\n#> 4 Fold01 52.8 49 42 Preprocessor1_Model1\n#> 5 Fold01 48.6 61 49 Preprocessor1_Model1\n#> 6 Fold01 29.8 66 40 Preprocessor1_Model1\n#> 7 Fold01 36.9 88 49 Preprocessor1_Model1\n#> # ℹ 3,742 more rows" + "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-2", + "href": "slides/intro-03-what-makes-a-model.html#your-turn-2", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_wflow chunk in your intro-03-classwork.qmd.\n\ntree_spec <- decision_tree() |> \n set_mode(\"classification\")\n\ntree_wflow <- workflow() |>\n add_formula(forested ~ .) |>\n add_model(tree_spec)\n\ntree_wflow\n\nEdit this code to make a workflow with your own model of choice.\n\nExtension/Challenge: Other than formulas, what kinds of preprocessors are supported?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#calibration-plot", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#calibration-plot", - "title": "1 - Feature Engineering", - "section": "Calibration Plot ", - "text": "Calibration Plot \n\nlibrary(probably)\n\ncal_plot_regression(hotel_lm_res, alpha = 1 / 5)" + "objectID": "slides/intro-03-what-makes-a-model.html#predict-with-your-model", + "href": "slides/intro-03-what-makes-a-model.html#predict-with-your-model", + "title": "3 - What makes a model?", + "section": "Predict with your model ", + "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(forested ~ ., tree_spec) |> \n fit(data = forested_train)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "title": "1 - Feature Engineering", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 unique companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor (in the extra materials)" + "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-3", + "href": "slides/intro-03-what-makes-a-model.html#your-turn-3", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = forested_test)\nWhat do you notice about the structure of the result?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#per-agent-statistics", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#per-agent-statistics", - "title": "1 - Feature Engineering", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-4", + "href": "slides/intro-03-what-makes-a-model.html#your-turn-4", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = forested_test)\nHow does the output compare to the output from predict()?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#collapsing-factor-levels", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#collapsing-factor-levels", - "title": "1 - Feature Engineering", - "section": "Collapsing factor levels ", - "text": "Collapsing factor levels \nThere is a recipe step that will redefine factor levels based on their frequency in the training set:\n\nhotel_other_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_other(agent, threshold = 0.001) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nUsing this code, 34 agents (out of 98) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." + "objectID": "slides/intro-03-what-makes-a-model.html#understand-your-model", + "href": "slides/intro-03-what-makes-a-model.html#understand-your-model", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#does-othering-help", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#does-othering-help", - "title": "1 - Feature Engineering", - "section": "Does othering help? ", - "text": "Does othering help? \n\nhotel_other_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hotel_other_rec)\n\nhotel_other_res <-\n hotel_other_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.213 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00341 Preprocessor1_Model1\n\nAbout the same MAE and much faster to complete.\nNow let’s look at a more sophisticated tool called effect feature hashing." + "objectID": "slides/intro-03-what-makes-a-model.html#understand-your-model-1", + "href": "slides/intro-03-what-makes-a-model.html#understand-your-model-1", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit |>\n extract_fit_engine() |>\n rpart.plot()\n\nYou can extract_*() several components of your fitted workflow.\n\n⚠️ Never predict() with any extracted components!\n\nroundint = FALSE is only to quiet a warning" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing", - "title": "1 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nBetween agent and company, simple dummy variables would create 198 new columns (that are mostly zeros).\nAnother option is to have a binary indicator that combines some levels of these variables.\nFeature hashing (for more see FES, SMLTAR, and TMwR):\n\nuses the character values of the levels\nconverts them to integer hash values\nuses the integers to assign them to a specific indicator column." + "objectID": "slides/intro-03-what-makes-a-model.html#understand-your-model-2", + "href": "slides/intro-03-what-makes-a-model.html#understand-your-model-2", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-1", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-1", - "title": "1 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nSuppose we want to use 32 indicator variables for agent.\nFor a agent with value “Max_Kuhn”, a hashing function converts it to an integer (say 210397726).\nTo assign it to one of the 32 columns, we would use modular arithmetic to assign it to a column:\n\n# For \"Max_Kuhn\" put a '1' in column: \n210397726 %% 32\n#> [1] 30\n\nHash functions are meant to emulate randomness." + "objectID": "slides/intro-03-what-makes-a-model.html#your-turn-5", + "href": "slides/intro-03-what-makes-a-model.html#your-turn-5", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\n\nExtract the model engine object from your fitted workflow and check it out.\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-pros", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-pros", - "title": "1 - Feature Engineering", - "section": "Feature Hashing Pros", - "text": "Feature Hashing Pros\n\nThe procedure will automatically work on new values of the predictors.\nIt is fast.\n“Signed” hashes add a sign to help avoid aliasing." + "objectID": "slides/intro-03-what-makes-a-model.html#the-whole-game---status-update", + "href": "slides/intro-03-what-makes-a-model.html#the-whole-game---status-update", + "title": "3 - What makes a model?", + "section": "The whole game - status update", + "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-cons", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-cons", - "title": "1 - Feature Engineering", - "section": "Feature Hashing Cons", - "text": "Feature Hashing Cons\n\nThere is no real logic behind which factor levels are combined.\nWe don’t know how many columns to add (more in the next section).\nSome columns may have all zeros.\nIf a indicator column is important to the model, we can’t easily determine why.\n\n\nThe signed hash make it slightly more possible to differentiate between confounded levels" + "objectID": "slides/intro-05-tuning-models.html#tuning-parameters", + "href": "slides/intro-05-tuning-models.html#tuning-parameters", + "title": "5 - Tuning models", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes", - "title": "1 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \nThe textrecipes package has a step that can be added to the recipe:\n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nhotel_hash_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hash_rec)" + "objectID": "slides/intro-05-tuning-models.html#optimize-tuning-parameters", + "href": "slides/intro-05-tuning-models.html#optimize-tuning-parameters", + "title": "5 - Tuning models", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", - "title": "1 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \n\nhotel_hash_res <-\n hotel_hash_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_hash_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.239 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00324 Preprocessor1_Model1\n\nAbout the same performance but now we can handle new values." + "objectID": "slides/intro-05-tuning-models.html#optimize-tuning-parameters-1", + "href": "slides/intro-05-tuning-models.html#optimize-tuning-parameters-1", + "title": "5 - Tuning models", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#debugging-a-recipe", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#debugging-a-recipe", - "title": "1 - Feature Engineering", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. hash_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "slides/intro-05-tuning-models.html#specifying-tuning-parameters", + "href": "slides/intro-05-tuning-models.html#specifying-tuning-parameters", + "title": "5 - Tuning models", + "section": "Specifying tuning parameters", + "text": "Specifying tuning parameters\nLet’s take our previous random forest workflow and tag for tuning the minimum number of data points in each node:\n\nrf_spec <- rand_forest(min_n = tune()) |> \n set_mode(\"classification\")\n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#example", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#example", - "title": "1 - Feature Engineering", - "section": "Example ", - "text": "Example \n\nhash_rec_fit <- prep(hash_rec)\n\n# Get the transformation coefficient\ntidy(hash_rec_fit, number = 1)\n\n# Get the processed data\nbake(hash_rec_fit, hotel_train %>% slice(1:3), contains(\"_agent_\"))" + "objectID": "slides/intro-05-tuning-models.html#try-out-multiple-values", + "href": "slides/intro-05-tuning-models.html#try-out-multiple-values", + "title": "5 - Tuning models", + "section": "Try out multiple values", + "text": "Try out multiple values\ntune_grid() works similar to fit_resamples() but covers multiple parameter values:\n\nset.seed(22)\nrf_res <- tune_grid(\n rf_wflow,\n forested_folds,\n grid = 5\n)" }, { - "objectID": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#more-on-recipes", - "href": "archive/2023-09-posit-conf/advanced-02-feature-engineering.html#more-on-recipes", - "title": "1 - Feature Engineering", - "section": "More on recipes", - "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." + "objectID": "slides/intro-05-tuning-models.html#compare-results", + "href": "slides/intro-05-tuning-models.html#compare-results", + "title": "5 - Tuning models", + "section": "Compare results", + "text": "Compare results\nInspecting results and selecting the best-performing hyperparameter(s):\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 40 roc_auc binary 0.762 10 0.0103 pre0_mod5_post0\n#> 2 31 roc_auc binary 0.761 10 0.0100 pre0_mod4_post0\n#> 3 22 roc_auc binary 0.760 10 0.00997 pre0_mod3_post0\n#> 4 12 roc_auc binary 0.758 10 0.0102 pre0_mod2_post0\n#> 5 3 roc_auc binary 0.755 10 0.00980 pre0_mod1_post0\n\nbest_parameter <- select_best(rf_res)\nbest_parameter\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 40 pre0_mod5_post0\n\ncollect_metrics() and autoplot() are also available." }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---setup", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---setup", - "title": "3 - Grid Search via Racing", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/intro-05-tuning-models.html#the-final-fit", + "href": "slides/intro-05-tuning-models.html#the-final-fit", + "title": "5 - Tuning models", + "section": "The final fit", + "text": "The final fit\n\nrf_wflow <- finalize_workflow(rf_wflow, best_parameter)\n\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.764 pre0_mod0_post0\n#> 2 roc_auc binary 0.764 pre0_mod0_post0\n#> 3 brier_class binary 0.161 pre0_mod0_post0" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---data-usage", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---data-usage", - "title": "3 - Grid Search via Racing", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "slides/intro-05-tuning-models.html#your-turn", + "href": "slides/intro-05-tuning-models.html#your-turn", + "title": "5 - Tuning models", + "section": "Your turn", + "text": "Your turn\n\nModify your model workflow to tune one or more parameters.\nUse grid search to find the best parameter(s).\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---boosting-model", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#previously---boosting-model", - "title": "3 - Grid Search via Racing", - "section": "Previously - Boosting Model ", - "text": "Previously - Boosting Model \n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "slides/intro-extra-recipes.html#looking-at-the-predictors", + "href": "slides/intro-extra-recipes.html#looking-at-the-predictors", + "title": "Extras - Recipes", + "section": "Looking at the predictors", + "text": "Looking at the predictors\n\nforested_train\n#> # A tibble: 8,749 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 1997 66 82 10 Tree 12.2 1315 18.4 1.88 24.9 11.8 121 1888 66 -84.9 32.4\n#> 2 No 1997 284 -99 58 Tree 10.3 1236 16.1 -0.26 22.3 9.92 68 1586 80 -85.0 34.1\n#> 3 Yes 2022 130 86 15 Tree 11.8 1194 17.6 1.3 24.1 11.2 61 1753 96 -83.0 33.1\n#> 4 Yes 2021 202 -55 3 Tree 10.7 1235 16.6 0.05 23.0 10.2 72 1682 65 -83.0 33.9\n#> 5 Yes 1995 75 -89 1 Tree 13.8 1256 19.2 3.63 25.5 12.9 57 1796 88 -83.4 31.3\n#> 6 No 1995 110 -53 5 Tree 12.4 1236 18.6 2.53 24.8 12.4 102 1835 51 -83.9 32.2\n#> 7 Yes 2022 111 73 12 Tree 11.5 1168 17.4 1 24.0 10.9 67 1772 84 -82.2 33.6\n#> 8 Yes 1997 230 96 14 Tree 9.98 1373 15.4 -1.35 21.8 9.03 46 1552 68 -85.3 34.8\n#> 9 Yes 2002 160 -88 13 Tree 11.1 1219 16.9 0.07 23.6 10.2 53 1731 95 -83.2 33.6\n#> 10 Yes 2020 39 9 6 Tree 13.9 1237 19.2 3.25 25.6 12.8 58 1812 86 -82.0 31.7\n#> # ℹ 8,739 more rows\n#> # ℹ 2 more variables: land_type <fct>, county <fct>" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#making-grid-search-more-efficient", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#making-grid-search-more-efficient", - "title": "3 - Grid Search via Racing", - "section": "Making Grid Search More Efficient", - "text": "Making Grid Search More Efficient\nIn the last section, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\nAlso, for some models, we can fit far fewer models than the number that are being evaluated.\n\nFor boosting, a model with X trees can often predict on candidates with less than X trees.\n\nBoth of these methods can lead to enormous speed-ups." + "objectID": "slides/intro-extra-recipes.html#working-with-other-models", + "href": "slides/intro-extra-recipes.html#working-with-other-models", + "title": "Extras - Recipes", + "section": "Working with other models", + "text": "Working with other models\nSome models can’t handle non-numeric data\n\nLinear Regression\nK Nearest Neighbors\n\n\n\nSome models struggle if numeric predictors aren’t scaled\n\nK Nearest Neighbors\nAnything using gradient descent" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#model-racing", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#model-racing", - "title": "3 - Grid Search via Racing", - "section": "Model Racing", - "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models but only for a few resamples.\nDetermine which candidates have a low probability of being selected.\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain)\n\nThis can result in fitting a small number of models." + "objectID": "slides/intro-extra-recipes.html#types-of-needed-preprocessing", + "href": "slides/intro-extra-recipes.html#types-of-needed-preprocessing", + "title": "Extras - Recipes", + "section": "Types of needed preprocessing", + "text": "Types of needed preprocessing\n\nDo qualitative predictors require a numeric encoding?\nShould columns with a single unique value be removed?\nDoes the model struggle with missing data?\nDoes the model struggle with correlated predictors?\nShould predictors be centered and scaled?\nIs it helpful to transform predictors to be more symmetric?\n\n\nhttps://www.tmwr.org/pre-proc-table.html" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#discarding-candidates", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#discarding-candidates", - "title": "3 - Grid Search via Racing", - "section": "Discarding Candidates", - "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large difference between resamples in the results." + "objectID": "slides/intro-extra-recipes.html#two-types-of-preprocessing", + "href": "slides/intro-extra-recipes.html#two-types-of-preprocessing", + "title": "Extras - Recipes", + "section": "Two types of preprocessing", + "text": "Two types of preprocessing" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#resampling-results-non-racing", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#resampling-results-non-racing", - "title": "3 - Grid Search via Racing", - "section": "Resampling Results (Non-Racing)", - "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." + "objectID": "slides/intro-extra-recipes.html#two-types-of-preprocessing-1", + "href": "slides/intro-extra-recipes.html#two-types-of-preprocessing-1", + "title": "Extras - Recipes", + "section": "Two types of preprocessing", + "text": "Two types of preprocessing" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#are-candidates-different", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#are-candidates-different", - "title": "3 - Grid Search via Racing", - "section": "Are Candidates Different?", - "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval is (0.99, 2.8), indicating that candidate number 2 has smaller error.\nWhat if we were to compare each model candidate to the current best at each resample?\nOne shows superiority when 4 resamples have been evaluated." + "objectID": "slides/intro-extra-recipes.html#general-definitions", + "href": "slides/intro-extra-recipes.html#general-definitions", + "title": "Extras - Recipes", + "section": "General definitions", + "text": "General definitions\n\nData preprocessing is what you do to make your model successful.\nFeature engineering is what you do to the original predictors to make the model do the least work to perform great." }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#evaluating-differences-in-candidates", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#evaluating-differences-in-candidates", - "title": "3 - Grid Search via Racing", - "section": "Evaluating Differences in Candidates", - "text": "Evaluating Differences in Candidates" + "objectID": "slides/intro-extra-recipes.html#working-with-dates", + "href": "slides/intro-extra-recipes.html#working-with-dates", + "title": "Extras - Recipes", + "section": "Working with dates", + "text": "Working with dates\nDatetime variables are automatically converted to an integer if given as a raw predictor. To avoid this, it can be re-encoded as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nLeap year\nIndicators for holidays" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#interim-analysis-of-results", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#interim-analysis-of-results", - "title": "3 - Grid Search via Racing", - "section": "Interim Analysis of Results", - "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order.\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." + "objectID": "slides/intro-extra-recipes.html#two-types-of-transformations", + "href": "slides/intro-extra-recipes.html#two-types-of-transformations", + "title": "Extras - Recipes", + "section": "Two types of transformations", + "text": "Two types of transformations\n\n\n\nStatic\n\nSquare root, log, inverse\nDummies for known levels\nDate time extractions\n\n\nTrained\n\nCentering & scaling\nImputation\nPCA\nAnything for unknown factor levels\n\n\n\nTrained methods need to calculate sufficient information to be applied again." }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#racing", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#racing", - "title": "3 - Grid Search via Racing", - "section": "Racing ", - "text": "Racing \n\n# Let's use a larger grid\nset.seed(8945)\nlgbm_grid <- \n lgbm_param %>% \n grid_latin_hypercube(size = 50)\n\nlibrary(finetune)\n\nset.seed(9)\nlgbm_race_res <-\n lgbm_wflow %>%\n tune_race_anova(\n resamples = hotel_rs,\n grid = lgbm_grid, \n metrics = reg_metrics\n )\n\nThe syntax and helper functions are extremely similar to those shown for tune_grid()." + "objectID": "slides/intro-extra-recipes.html#the-recipes-package", + "href": "slides/intro-extra-recipes.html#the-recipes-package", + "title": "Extras - Recipes", + "section": "The recipes package", + "text": "The recipes package\n\n\nModular + extensible\nWorks well with pipes ,|> and %>%\nDeferred evaluation\nIsolates test data from training data\nCan do things formulas can’t" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#racing-results", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#racing-results", - "title": "3 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\nshow_best(lgbm_race_res, metric = \"mae\")\n#> # A tibble: 2 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1516 7 0.0421 176 12 mae standard 9.60 10 0.181 Preprocessor42_Model1\n#> 2 1014 5 0.0791 35 181 mae standard 9.61 10 0.179 Preprocessor06_Model1" + "objectID": "slides/intro-extra-recipes.html#how-to-write-a-recipe", + "href": "slides/intro-extra-recipes.html#how-to-write-a-recipe", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())" }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#racing-results-1", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#racing-results-1", - "title": "3 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\n\nOnly 171 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_race_res) + \n scale_x_continuous(breaks = pretty_breaks())" + "objectID": "slides/intro-extra-recipes.html#how-to-write-a-recipe-1", + "href": "slides/intro-extra-recipes.html#how-to-write-a-recipe-1", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())\n\n\nStart by calling recipe() to denote the data source and variables used." }, { - "objectID": "archive/2023-09-posit-conf/advanced-04-racing.html#your-turn", - "href": "archive/2023-09-posit-conf/advanced-04-racing.html#your-turn", - "title": "3 - Grid Search via Racing", - "section": "Your turn", - "text": "Your turn\n\nRun tune_race_anova() with a different seed.\nDid you get the same or similar results?\n\n\n\n\n−+\n10:00" + "objectID": "slides/intro-extra-recipes.html#how-to-write-a-recipe-2", + "href": "slides/intro-extra-recipes.html#how-to-write-a-recipe-2", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())\n\n\nSpecify what actions to take by adding step_*()s." }, { - "objectID": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#your-turn", - "href": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#your-turn", - "title": "6 - Wrapping up", - "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" + "objectID": "slides/intro-extra-recipes.html#how-to-write-a-recipe-3", + "href": "slides/intro-extra-recipes.html#how-to-write-a-recipe-3", + "title": "Extras - Recipes", + "section": "How to write a recipe", + "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>   step_normalize(all_numeric_predictors())\n\n\nUse {tidyselect} and recipes-specific selectors to denote affected variables." }, { - "objectID": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#resources-to-keep-learning", - "href": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#resources-to-keep-learning", - "title": "6 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" + "objectID": "slides/intro-extra-recipes.html#using-a-recipe", + "href": "slides/intro-extra-recipes.html#using-a-recipe", + "title": "Extras - Recipes", + "section": "Using a recipe", + "text": "Using a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>   step_normalize(all_numeric_predictors())\n\n\nSave the recipe we like so that we can use it in various places, e.g., with different models." }, { - "objectID": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#feedback", - "href": "archive/2023-09-posit-conf/advanced-06-wrapping-up.html#feedback", - "title": "6 - Wrapping up", - "section": "Feedback", - "text": "Feedback\n\nPlease go to http://pos.it/conf-workshop-survey. Your feedback is crucial!\nData from the survey informs curriculum and format decisions for future conf workshops, and we really appreciate you taking the time to provide it." + "objectID": "slides/intro-extra-recipes.html#using-a-recipe-with-workflows", + "href": "slides/intro-extra-recipes.html#using-a-recipe-with-workflows", + "title": "Extras - Recipes", + "section": "Using a recipe with workflows", + "text": "Using a recipe with workflows\nRecipes are typically combined with a model in a workflow() object:\n\n\nforested_wflow <- workflow() |>\n  add_recipe(forested_rec) |>\n  add_model(linear_reg())" }, { - "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#previously---setup", - "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#previously---setup", - "title": "Extras - Effect Encodings", - "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "slides/intro-extra-recipes.html#recipes-are-estimated", + "href": "slides/intro-extra-recipes.html#recipes-are-estimated", + "title": "Extras - Recipes", + "section": "Recipes are estimated", + "text": "Recipes are estimated\nEvery preprocessing step in a recipe that involved calculations uses the training set. For example:\n\nLevels of a factor\nDetermination of zero-variance\nNormalization\nFeature extraction\n\nOnce a recipe is added to a workflow, this occurs when fit() is called." }, { - "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#previously---data-usage", - "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#previously---data-usage", - "title": "Extras - Effect Encodings", - "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "slides/intro-extra-recipes.html#debugging-a-recipe", + "href": "slides/intro-extra-recipes.html#debugging-a-recipe", + "title": "Extras - Recipes", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. forested_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4.\n\n\n\n\nAnother function, bake(), is analogous to predict(), and gives you the processed data back." }, { - "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "title": "Extras - Effect Encodings", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor" + "objectID": "slides/intro-extra-recipes.html#your-turn", + "href": "slides/intro-extra-recipes.html#your-turn", + "title": "Extras - Recipes", + "section": "Your turn", + "text": "Your turn\n\n\nTake the recipe and prep() then bake() it to see what the resulting data set looks like.\nTry removing steps to see how the result changes.\n\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#per-agent-statistics", - "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#per-agent-statistics", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "slides/intro-extra-recipes.html#printing-a-recipe", + "href": "slides/intro-extra-recipes.html#printing-a-recipe", + "title": "Extras - Recipes", + "section": "Printing a recipe", + "text": "Printing a recipe\n\nforested_rec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Operations\n#> • Dummy variables from: all_nominal_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Log transformation on: canopy_cover\n#> • Centering and scaling for: all_numeric_predictors()" }, { - "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#what-is-an-effect-encoding", - "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#what-is-an-effect-encoding", - "title": "Extras - Effect Encodings", - "section": "What is an effect encoding?", - "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <fct> <int>\n#> 1 52.7 cynthia_worsley 1\n#> 2 51.8 carlos_bryant 2\n#> 3 53.8 lance_hitchcock 3\n#> 4 51.8 lance_hitchcock 4\n#> 5 46.8 cynthia_worsley 5\n#> 6 54.7 charles_najera 6\n#> 7 46.8 cynthia_worsley 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <dbl> <int>\n#> 1 52.7 88.5 1\n#> 2 51.8 89.5 2\n#> 3 53.8 79.8 3\n#> 4 51.8 79.8 4\n#> 5 46.8 88.5 5\n#> 6 54.7 109. 6\n#> 7 46.8 88.5 7\n\n\nThe agent column is replaced with an estimate of the ADR." + "objectID": "slides/intro-extra-recipes.html#prepping-a-recipe", + "href": "slides/intro-extra-recipes.html#prepping-a-recipe", + "title": "Extras - Recipes", + "section": "Prepping a recipe", + "text": "Prepping a recipe\n\nprep(forested_rec)\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Training information\n#> Training data contained 8749 data points and no incomplete rows.\n#> \n#> ── Operations\n#> • Dummy variables from: tree_no_tree, land_type, county | Trained\n#> • Zero variance filter removed: <none> | Trained\n#> • Log transformation on: canopy_cover | Trained\n#> • Centering and scaling for: year elevation, ... | Trained" }, { - "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#per-agent-statistics-again", - "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#per-agent-statistics-again", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics again", - "text": "Per-agent statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these means use partial pooling.\nPooling borrows strength across agents and shrinks extreme values towards the mean for agents with very few transations\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for agents with fewer reservations by shrinking the estimate to the overall ADR mean" + "objectID": "slides/intro-extra-recipes.html#baking-a-recipe", + "href": "slides/intro-extra-recipes.html#baking-a-recipe", + "title": "Extras - Recipes", + "section": "Baking a recipe", + "text": "Baking a recipe\n\nprep(forested_rec) |>\n bake(new_data = forested_train)\n#> # A tibble: 8,749 × 177\n#> year elevation eastness roughness dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat forested\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> \n#> 1 -1.15 -0.521 1.13 0.0463 -0.232 0.178 0.172 -0.107 0.338 -0.00909 3.10 1.01 0.407 -1.44 -0.0184 Yes \n#> 2 -1.15 1.10 -1.50 3.91 -1.39 -0.568 -1.31 -1.33 -1.28 -1.28 0.0408 -0.941 0.652 -1.51 1.36 No \n#> 3 0.892 -0.0440 1.19 0.449 -0.422 -0.965 -0.305 -0.438 -0.170 -0.438 -0.363 0.140 0.884 0.331 0.556 Yes \n#> 4 0.810 0.493 -0.858 -0.518 -1.11 -0.577 -1.01 -1.15 -0.878 -1.10 0.272 -0.320 0.388 0.331 1.26 Yes \n#> 5 -1.31 -0.454 -1.35 -0.679 0.748 -0.379 0.701 0.893 0.683 0.702 -0.594 0.418 0.773 -0.0643 -0.992 Yes \n#> 6 -1.31 -0.193 -0.829 -0.357 -0.0546 -0.568 0.331 0.264 0.281 0.366 2.00 0.670 0.0802 -0.468 -0.237 No \n#> 7 0.892 -0.186 1.00 0.207 -0.631 -1.21 -0.444 -0.609 -0.264 -0.619 -0.0169 0.262 0.714 1.01 0.929 Yes \n#> 8 -1.15 0.702 1.34 0.369 -1.56 0.726 -1.77 -1.95 -1.60 -1.87 -1.23 -1.16 0.445 -1.77 2.03 Yes \n#> 9 -0.740 0.180 -1.34 0.288 -0.858 -0.729 -0.801 -1.14 -0.515 -1.08 -0.825 -0.00273 0.870 0.122 0.988 Yes \n#> 10 0.729 -0.722 0.0719 -0.276 0.816 -0.559 0.708 0.676 0.745 0.635 -0.536 0.521 0.744 1.22 -0.653 Yes \n#> # ℹ 8,739 more rows\n#> # ℹ 161 more variables: tree_no_tree_No.tree <dbl>, land_type_Non.tree.vegetation <dbl>, land_type_Tree <dbl>, county_Atkinson <dbl>, county_Bacon <dbl>, county_Baker <dbl>, county_Baldwin <dbl>,\n#> # county_Banks <dbl>, county_Barrow <dbl>, county_Bartow <dbl>, county_Ben.Hill <dbl>, county_Berrien <dbl>, county_Bibb <dbl>, county_Bleckley <dbl>, county_Brantley <dbl>, county_Brooks <dbl>,\n#> # county_Bryan <dbl>, county_Bulloch <dbl>, county_Burke <dbl>, county_Butts <dbl>, county_Calhoun <dbl>, county_Camden <dbl>, county_Candler <dbl>, county_Carroll <dbl>, county_Catoosa <dbl>,\n#> # county_Charlton <dbl>, county_Chatham <dbl>, county_Chattahoochee <dbl>, county_Chattooga <dbl>, county_Cherokee <dbl>, county_Clarke <dbl>, county_Clay <dbl>, county_Clayton <dbl>,\n#> # county_Clinch <dbl>, county_Cobb <dbl>, county_Coffee <dbl>, county_Colquitt <dbl>, county_Columbia <dbl>, county_Cook <dbl>, county_Coweta <dbl>, county_Crawford <dbl>, county_Crisp <dbl>,\n#> # county_Dade <dbl>, county_Dawson <dbl>, county_Decatur <dbl>, county_DeKalb <dbl>, county_Dodge <dbl>, county_Dooly <dbl>, county_Dougherty <dbl>, county_Douglas <dbl>, county_Early <dbl>, …" }, { - "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#partial-pooling", - "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#partial-pooling", - "title": "Extras - Effect Encodings", - "section": "Partial pooling", - "text": "Partial pooling" + "objectID": "slides/intro-extra-recipes.html#tidying-a-recipe", + "href": "slides/intro-extra-recipes.html#tidying-a-recipe", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\nOnce a recipe as been estimated, there are various bits of information saved in it.\n\nThe tidy() function can be used to get specific results from the recipe." }, { - "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#agent-effects", - "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#agent-effects", - "title": "Extras - Effect Encodings", - "section": "Agent effects ", - "text": "Agent effects \n\nlibrary(embed)\n\nhotel_effect_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_lencode_mixed(agent, company, outcome = vars(avg_price_per_room)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." + "objectID": "slides/intro-extra-recipes.html#your-turn-1", + "href": "slides/intro-extra-recipes.html#your-turn-1", + "title": "Extras - Recipes", + "section": "Your turn", + "text": "Your turn\n\nTake a prepped recipe and use the tidy() function on it.\nUse the number argument to inspect different steps.\n\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/extras-effect-encodings.html#effect-encoding-results", - "href": "archive/2023-09-posit-conf/extras-effect-encodings.html#effect-encoding-results", - "title": "Extras - Effect Encodings", - "section": "Effect encoding results ", - "text": "Effect encoding results \n\nhotel_effect_wflow <-\n workflow() %>%\n add_model(linear_reg()) %>% \n update_recipe(hotel_effect_rec)\n\nreg_metrics <- metric_set(mae, rsq)\n\nhotel_effect_res <-\n hotel_effect_wflow %>%\n fit_resamples(hotel_rs, metrics = reg_metrics)\n\ncollect_metrics(hotel_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.8 10 0.189 Preprocessor1_Model1\n#> 2 rsq standard 0.870 10 0.00357 Preprocessor1_Model1\n\nSlightly worse but it can handle new agents (if they occur)." + "objectID": "slides/intro-extra-recipes.html#tidying-a-recipe-1", + "href": "slides/intro-extra-recipes.html#tidying-a-recipe-1", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\n\nprep(forested_rec) |>\n tidy()\n#> # A tibble: 4 × 6\n#> number operation type trained skip id \n#> <int> <chr> <chr> <lgl> <lgl> <chr> \n#> 1 1 step dummy TRUE FALSE dummy_hIEnQ \n#> 2 2 step zv TRUE FALSE zv_ZrQBx \n#> 3 3 step log TRUE FALSE log_6es7X \n#> 4 4 step normalize TRUE FALSE normalize_XsIxb" }, { - "objectID": "archive/2023-09-posit-conf/extras-vetiver.html#deploying-a-model", - "href": "archive/2023-09-posit-conf/extras-vetiver.html#deploying-a-model", - "title": "Extras - Model deployment", - "section": "Deploying a model ", - "text": "Deploying a model \nWe have a decision tree, tree_fit, to model whether or not a taxi trip in Chicago included a tip or not.\nHow do we use our model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"taxi\")\nv\n#> \n#> ── taxi ─ <bundled_workflow> model for deployment \n#> A rpart classification modeling workflow using 6 features\n\nLearn more at https://vetiver.rstudio.com" + "objectID": "slides/intro-extra-recipes.html#tidying-a-recipe-2", + "href": "slides/intro-extra-recipes.html#tidying-a-recipe-2", + "title": "Extras - Recipes", + "section": "Tidying a recipe", + "text": "Tidying a recipe\n\nprep(forested_rec) |>\n tidy(number = 1)\n#> # A tibble: 161 × 3\n#> terms columns id \n#> <chr> <chr> <chr> \n#> 1 tree_no_tree No tree dummy_hIEnQ\n#> 2 land_type Non-tree vegetation dummy_hIEnQ\n#> 3 land_type Tree dummy_hIEnQ\n#> 4 county Atkinson dummy_hIEnQ\n#> 5 county Bacon dummy_hIEnQ\n#> 6 county Baker dummy_hIEnQ\n#> 7 county Baldwin dummy_hIEnQ\n#> 8 county Banks dummy_hIEnQ\n#> 9 county Barrow dummy_hIEnQ\n#> 10 county Bartow dummy_hIEnQ\n#> # ℹ 151 more rows" }, { - "objectID": "archive/2023-09-posit-conf/extras-vetiver.html#deploy-your-model-1", - "href": "archive/2023-09-posit-conf/extras-vetiver.html#deploy-your-model-1", - "title": "Extras - Model deployment", - "section": "Deploy your model ", - "text": "Deploy your model \nHow do we use our model in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 4 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Users/simoncouch/Library/R/arm64/4.2/library/vetiver\n#> ├──/metadata (GET)\n#> ├──/ping (GET)\n#> ├──/predict (POST)\n#> └──/prototype (GET)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" + "objectID": "slides/intro-extra-recipes.html#using-a-recipe-in-tidymodels", + "href": "slides/intro-extra-recipes.html#using-a-recipe-in-tidymodels", + "title": "Extras - Recipes", + "section": "Using a recipe in tidymodels", + "text": "Using a recipe in tidymodels\nThe recommended way to use a recipe in tidymodels is to use it as part of a workflow().\n\nforested_wflow <- workflow() |> \n add_recipe(forested_rec) |> \n add_model(linear_reg())\n\nWhen used in this way, you don’t need to worry about prep() and bake() as it is handled for you." }, { - "objectID": "archive/2023-09-posit-conf/extras-vetiver.html#your-turn", - "href": "archive/2023-09-posit-conf/extras-vetiver.html#your-turn", - "title": "Extras - Model deployment", - "section": "Your turn", - "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n−+\n05:00" + "objectID": "slides/intro-extra-recipes.html#more-information", + "href": "slides/intro-extra-recipes.html#more-information", + "title": "Extras - Recipes", + "section": "More information", + "text": "More information\n\nhttps://recipes.tidymodels.org/\nhttps://www.tmwr.org/recipes.html" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#workshop-policies", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#workshop-policies", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#workshop-policies", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#workshop-policies", "title": "1 - Introduction", "section": "Workshop policies", - "text": "Workshop policies\n\nPlease do not photograph people wearing red lanyards\nThere are gender-neutral bathrooms located are among the Grand Suite Bathrooms\nThere are two meditation/prayer rooms: Grand Suite 2A and 2B\nA lactation room is located in Grand Suite 1\nThe meditation/prayer and lactation rooms are open\nSun - Tue 7:30am - 7:00pm, Wed 8:00am - 6:00pm" + "text": "Workshop policies\n\nPlease do not photograph people wearing red lanyards\nThere are gender neutral bathrooms near National Harbor rooms\nA meditation room is located at National Harbor 9 (8am - 5pm, Mon - Thurs)\nA lactation room is located at Potomac Dressing Room (8am - 5pm, Mon - Thurs)" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#workshop-policies-1", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#workshop-policies-1", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#workshop-policies-1", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#workshop-policies-1", "title": "1 - Introduction", "section": "Workshop policies", - "text": "Workshop policies\n\nPlease review the code of conduct and COVID policies, which apply to all workshops: https://posit.co/code-of-conduct/.\nCoC site has info on how to report a problem (in person, email, phone)" + "text": "Workshop policies\n\nPlease review the rstudio::conf code of conduct, which applies to all workshops: https://www.rstudio.com/conference/2022/2022-conf-code-of-conduct/\nCoC site has info on how to report a problem (in person, email, phone)\nYou are required to wear a mask that fully covers your mouth and nose at all times in all public spaces" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#who-are-you", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#who-are-you", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-you", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-you", "title": "1 - Introduction", "section": "Who are you?", "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#who-are-tidymodels", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#who-are-tidymodels", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-we", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-we", + "title": "1 - Introduction", + "section": "Who are we?", + "text": "Who are we?\n\n\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\n\nJulia Silge\nDavid Robinson\nDavis Vaughan" + }, + { + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-we-1", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#who-are-we-1", "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nSilvia Canelón is TAing today!\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "section": "Who are we?", + "text": "Who are we?\n\n\n\nKelly Bodwin\nMichael Chow\nPritam Dalal\nMatt Dancho\nJon Harmon\n\n\n\nMike Mahoney\nEdgar Ruiz\nAsmae Toumi\nQiushi Yan\n\n\n\nMany thanks to Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#asking-for-help", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#asking-for-help", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#asking-for-help", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#asking-for-help", "title": "1 - Introduction", "section": "Asking for help", "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#section-2", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#section-2", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#section-2", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#section-2", "title": "1 - Introduction", "section": "👀", "text": "👀" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#plan-for-this-workshop", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#plan-for-this-workshop", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#plan-for-this-workshop", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#plan-for-this-workshop", "title": "1 - Introduction", "section": "Plan for this workshop", - "text": "Plan for this workshop\n\nYour data budget\nWhat makes a model\nEvaluating models\nTuning models\n\n\nThis workshop will well-prepare folks going on to the Advanced tidymodels workshop, which will cover feature engineering and much more on hyperparameter tuning." + "text": "Plan for this workshop\n\nToday:\n\nYour data budget\nWhat makes a model\nEvaluating models\n\nTomorrow:\n\nFeature engineering\nTuning hyperparameters\nWrapping up!" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#section-3", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#section-3", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#section-3", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#section-3", "title": "1 - Introduction", "section": "", - "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free):\nCheck the workshop channel on Discord for the link!" + "text": "Introduce yourself to your neighbors 👋\n\n Log in to RStudio Cloud here (free):\nbit.ly/tidymodels-rstudioconf-2022" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning", "title": "1 - Introduction", "section": "What is machine learning?", "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning-1", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning-1", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning-1", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning-1", "title": "1 - Introduction", "section": "What is machine learning?", "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning-2", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-machine-learning-2", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning-2", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-machine-learning-2", "title": "1 - Introduction", "section": "What is machine learning?", "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#your-turn", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#your-turn", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#your-turn", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#your-turn", "title": "1 - Introduction", "section": "Your turn", - "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-tidymodels", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#what-is-tidymodels", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-tidymodels", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-is-tidymodels", "title": "1 - Introduction", "section": "What is tidymodels? ", - "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.1.1 ──\n#> ✔ broom 1.0.5 ✔ rsample 1.2.0\n#> ✔ dials 1.2.0 ✔ tibble 3.2.1\n#> ✔ dplyr 1.1.3 ✔ tidyr 1.3.0\n#> ✔ infer 1.0.5 ✔ tune 1.1.2\n#> ✔ modeldata 1.2.0 ✔ workflows 1.1.3\n#> ✔ parsnip 1.1.1 ✔ workflowsets 1.0.1\n#> ✔ purrr 1.0.2 ✔ yardstick 1.2.0\n#> ✔ recipes 1.0.8\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Learn how to get started at https://www.tidymodels.org/start/" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\nRoadmap for today\nMinimal version of predictive modeling process\nFeature engineering and tuning as iterative extensions" + "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.0.0 ──\n#> ✔ broom 1.0.0 ✔ rsample 1.0.0\n#> ✔ dials 1.0.0 ✔ tibble 3.1.8\n#> ✔ dplyr 1.0.9 ✔ tidyr 1.2.0\n#> ✔ infer 1.0.2 ✔ tune 1.0.0\n#> ✔ modeldata 1.0.0 ✔ workflows 1.0.0\n#> ✔ parsnip 1.0.0 ✔ workflowsets 1.0.0\n#> ✔ purrr 0.3.4 ✔ yardstick 1.0.0\n#> ✔ recipes 1.0.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Learn how to get started at https://www.tidymodels.org/start/" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-1", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-1", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#the-whole-game", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#the-whole-game", "title": "1 - Introduction", "section": "The whole game", - "text": "The whole game" + "text": "The whole game\n\nTomorrow we will walk through a case study in detail to illustrate feature engineering and model tuning.\nToday we will walk through the analysis at a higher level to show the model development process as a whole and give you an introduction to the data set.\nThe data are from the NHL where we want to predict whether a shot was on-goal or not! 🏒\nIt’s a good example to show how model development works." }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-2", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-2", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#shots-on-goal", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#shots-on-goal", "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." + "section": "Shots on goal", + "text": "Shots on goal" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-3", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-3", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#data-spending", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#data-spending", "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "section": "Data spending", + "text": "Data spending" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-4", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-4", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#a-first-model", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#a-first-model", "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "section": "A first model", + "text": "A first model" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-5", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-5", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#starting-point-logistic-regression", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#starting-point-logistic-regression", "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "section": "Starting point: logistic regression", + "text": "Starting point: logistic regression\n\nWe’ll start by using basic logistic regression to predict our binary outcome.\nOur first model will have 16 simple predictor columns.\nOne initial question: there are 640 players taking shots.\nFor logistic regression, do we convert these to binary indicators (a.k.a. “dummies”)?" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-6", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-6", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#basic-features-inc-dummy-variables", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#basic-features-inc-dummy-variables", "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "section": "Basic features (inc dummy variables)", + "text": "Basic features (inc dummy variables)" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-7", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#the-whole-game-7", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#different-player-encoding", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#different-player-encoding", "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "section": "Different player encoding", + "text": "Different player encoding" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#lets-install-some-packages", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#lets-install-some-packages", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#what-about-location", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#what-about-location", "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\",\n \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\",\n \"splines2\", \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \n \"vetiver\", \"remotes\")\n\ninstall.packages(pkgs)\n\n\n\n Or log in to Posit Cloud\nLink in our Discord channel!" + "section": "What about location", + "text": "What about location\nThe previous models used the x/y coordinates.\nAre there better ways to represent shot location?\nHow can we make location more usable for the model?" }, { - "objectID": "archive/2023-09-posit-conf/intro-01-introduction.html#our-versions", - "href": "archive/2023-09-posit-conf/intro-01-introduction.html#our-versions", + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-angle", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-angle", "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nR version 4.2.2 (2022-10-31), Quarto (1.4.104)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.2.1\n\n\nbroom\n1.0.5\n\n\ndials\n1.2.0\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.3\n\n\nembed\n1.1.2\n\n\nfinetune\n1.1.0\n\n\nggplot2\n3.4.3\n\n\nlightgbm\n3.3.5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nlme4\n1.1-34\n\n\nmodeldata\n1.2.0\n\n\nparsnip\n1.1.1\n\n\nplumber\n1.2.1\n\n\nprobably\n1.0.2\n\n\npurrr\n1.0.2\n\n\nranger\n0.15.1\n\n\nrecipes\n1.0.8\n\n\nremotes\n2.4.2.1\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nrpart\n4.1.19\n\n\nrpart.plot\n3.1.1\n\n\nrsample\n1.2.0\n\n\nrules\n1.0.2\n\n\nscales\n1.2.1\n\n\nsplines2\n0.5.1\n\n\nstacks\n1.0.2\n\n\ntext2vec\n0.6.3\n\n\ntextrecipes\n1.0.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.1.1\n\n\ntidyr\n1.3.0\n\n\ntune\n1.1.2\n\n\nvetiver\n0.2.4\n\n\nworkflows\n1.1.3\n\n\nworkflowsets\n1.0.1\n\n\nyardstick\n1.2.0" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglm for generalized linear model (e.g. logistic regression)\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-1", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-1", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-2", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-2", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-3", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-3", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-4", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-4", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-5", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-5", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-6", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-6", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-7", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-7", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-8", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-8", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-1", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-1", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a logistic regression model.\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/\n\n\nExtension/Challenge: Edit this code to use a different model. For example, try using a conditional inference tree as implemented in the partykit package by changing the engine - or try an entirely different model type!\n\n\n\n−+\n05:00" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#models-well-be-using-today", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#models-well-be-using-today", - "title": "3 - What makes a model?", - "section": "Models we’ll be using today", - "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-1", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-1", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-2", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-2", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nFind a sigmoid line that separates the two classes" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees-1", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees-1", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" + "section": "Add shot angle?", + "text": "Add shot angle?" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees-2", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#decision-trees-2", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-distance", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-distance", + "title": "1 - Introduction", + "section": "Add shot distance?", + "text": "Add shot distance?" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "title": "3 - What makes a model?", - "section": "All models are wrong, but some are useful!", - "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-behind-goal-line", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#add-shot-behind-goal-line", + "title": "1 - Introduction", + "section": "Add shot behind goal line?", + "text": "Add shot behind goal line?" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "title": "3 - What makes a model?", - "section": "Workflows bind preprocessors and models", - "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#nonlinear-terms-for-angle-and-distance", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#nonlinear-terms-for-angle-and-distance", + "title": "1 - Introduction", + "section": "Nonlinear terms for angle and distance", + "text": "Nonlinear terms for angle and distance" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#what-is-wrong-with-this", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#what-is-wrong-with-this", - "title": "3 - What makes a model?", - "section": "What is wrong with this?", - "text": "What is wrong with this?" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#try-another-model", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#try-another-model", + "title": "1 - Introduction", + "section": "Try another model", + "text": "Try another model" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#why-a-workflow", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#why-a-workflow", - "title": "3 - What makes a model?", - "section": "Why a workflow()? ", - "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering in Advanced tidymodels!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#switch-to-boosting-and-basic-features", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#switch-to-boosting-and-basic-features", + "title": "1 - Introduction", + "section": "Switch to boosting and basic features", + "text": "Switch to boosting and basic features" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-1", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-1", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\ntree_spec %>% \n fit(tip ~ ., data = taxi_train) \n#> parsnip model object\n#> \n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 yes (0.92300000 0.07700000) \n#> 2) distance>=14.12 2041 68 yes (0.96668300 0.03331700) *\n#> 3) distance< 14.12 5959 548 yes (0.90803826 0.09196174) \n#> 6) distance< 5.275 5419 450 yes (0.91695885 0.08304115) *\n#> 7) distance>=5.275 540 98 yes (0.81851852 0.18148148) \n#> 14) company=Chicago Independents,City Service,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc,other 478 68 yes (0.85774059 0.14225941) *\n#> 15) company=Flash Cab 62 30 yes (0.51612903 0.48387097) \n#> 30) dow=Thu 12 2 yes (0.83333333 0.16666667) *\n#> 31) dow=Sun,Mon,Tue,Wed,Fri,Sat 50 22 no (0.44000000 0.56000000) \n#> 62) distance>=11.77 14 4 yes (0.71428571 0.28571429) *\n#> 63) distance< 11.77 36 12 no (0.33333333 0.66666667) *" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#boosting-with-location-features", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#boosting-with-location-features", + "title": "1 - Introduction", + "section": "Boosting with location features", + "text": "Boosting with location features" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-2", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-2", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\nworkflow() %>%\n add_formula(tip ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = taxi_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 yes (0.92300000 0.07700000) \n#> 2) distance>=14.12 2041 68 yes (0.96668300 0.03331700) *\n#> 3) distance< 14.12 5959 548 yes (0.90803826 0.09196174) \n#> 6) distance< 5.275 5419 450 yes (0.91695885 0.08304115) *\n#> 7) distance>=5.275 540 98 yes (0.81851852 0.18148148) \n#> 14) company=Chicago Independents,City Service,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc,other 478 68 yes (0.85774059 0.14225941) *\n#> 15) company=Flash Cab 62 30 yes (0.51612903 0.48387097) \n#> 30) dow=Thu 12 2 yes (0.83333333 0.16666667) *\n#> 31) dow=Sun,Mon,Tue,Wed,Fri,Sat 50 22 no (0.44000000 0.56000000) \n#> 62) distance>=11.77 14 4 yes (0.71428571 0.28571429) *\n#> 63) distance< 11.77 36 12 no (0.33333333 0.66666667) *" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#choose-wisely", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#choose-wisely", + "title": "1 - Introduction", + "section": "Choose wisely…", + "text": "Choose wisely…" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-3", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-3", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\nworkflow(tip ~ ., tree_spec) %>% \n fit(data = taxi_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 yes (0.92300000 0.07700000) \n#> 2) distance>=14.12 2041 68 yes (0.96668300 0.03331700) *\n#> 3) distance< 14.12 5959 548 yes (0.90803826 0.09196174) \n#> 6) distance< 5.275 5419 450 yes (0.91695885 0.08304115) *\n#> 7) distance>=5.275 540 98 yes (0.81851852 0.18148148) \n#> 14) company=Chicago Independents,City Service,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc,other 478 68 yes (0.85774059 0.14225941) *\n#> 15) company=Flash Cab 62 30 yes (0.51612903 0.48387097) \n#> 30) dow=Thu 12 2 yes (0.83333333 0.16666667) *\n#> 31) dow=Sun,Mon,Tue,Wed,Fri,Sat 50 22 no (0.44000000 0.56000000) \n#> 62) distance>=11.77 14 4 yes (0.71428571 0.28571429) *\n#> 63) distance< 11.77 36 12 no (0.33333333 0.66666667) *" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#finalize-and-verify", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#finalize-and-verify", + "title": "1 - Introduction", + "section": "Finalize and verify", + "text": "Finalize and verify" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-2", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-2", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code to make a workflow with your own model of choice.\n\nExtension/Challenge: Other than formulas, what kinds of preprocessors are supported?\n\n\n\n−+\n05:00" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#and-so-on", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#and-so-on", + "title": "1 - Introduction", + "section": "… and so on", + "text": "… and so on\nOnce we find an acceptable model and feature set, the process is to\n\nConfirm our results on the test set.\nDocument the data and model development process.\nDeploy, monitor, etc." }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#predict-with-your-model", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#predict-with-your-model", - "title": "3 - What makes a model?", - "section": "Predict with your model ", - "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(tip ~ ., tree_spec) %>% \n fit(data = taxi_train)" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#lets-install-some-packages", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of RStudio Cloud:\n\ninstall.packages(c(\"DALEXtra\", \"doParallel\", \"embed\", \"forcats\",\n \"lme4\", \"ranger\", \"remotes\", \"rpart\", \n \"rpart.plot\", \"stacks\", \"tidymodels\",\n \"vetiver\", \"xgboost\"))\n\nremotes::install_github(\"topepo/ongoal@v0.0.2\")\n\n\n\n Or log in to RStudio Cloud:\nbit.ly/tidymodels-rstudioconf-2022" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-3", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-3", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = taxi_test)\nWhat do you get?\n\n\n\n−+\n03:00" + "objectID": "archive/2022-07-RStudio-conf/01-introduction.html#our-versions", + "href": "archive/2022-07-RStudio-conf/01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nbroom (1.0.0, CRAN), DALEX (2.4.0, CRAN), DALEXtra (2.2.0, CRAN), dials (1.0.0, CRAN), doParallel (1.0.17, CRAN), dplyr (1.0.9, CRAN), embed (1.0.0, CRAN), ggplot2 (3.3.6, CRAN), modeldata (1.0.0, CRAN), ongoal (0.0.2, Github (topepo/ongoal@02cd6b233), parsnip (1.0.0, CRAN), purrr (0.3.4, CRAN), ranger (0.13.1, CRAN), recipes (1.0.1, local), rpart (4.1.16, CRAN), rpart.plot (3.1.1, CRAN), rsample (1.0.0, CRAN), scales (1.2.0, CRAN), stacks (0.2.3, CRAN), tibble (3.1.8, CRAN), tidymodels (1.0.0, CRAN), tidyr (1.2.0, CRAN), tune (1.0.0, CRAN), vetiver (0.1.5, CRAN), workflows (1.0.0, CRAN), workflowsets (1.0.0, CRAN), xgboost (1.6.0.1, CRAN), and yardstick (1.0.0, CRAN)" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-4", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-4", + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn", "title": "3 - What makes a model?", "section": "Your turn", - "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = taxi_test)\nWhat do you get?\n\n\n\n−+\n03:00" + "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n03:00\n\n\n\n\n\nlm for linear model\nglm for generalized linear model (e.g. logistic regression)\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model", + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model", "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?" + "section": "To specify a model ", + "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-1", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-1", + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-1", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-1", "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\n⚠️ Never predict() with any extracted components!\n\nroundint = FALSE is only to quiet a warning" + "section": "To specify a model ", + "text": "To specify a model \n\nlinear_reg()\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: lm\n\n\nModels have default engines" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-2", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-2", + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-2", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-2", "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-5", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#your-turn-5", + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-3", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-3", "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\n\nExtract the model engine object from your fitted workflow and check it out.\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" + "section": "To specify a model ", + "text": "To specify a model \n\nlinear_reg() %>%\n set_engine(\"glmnet\")\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: glmnet" }, { - "objectID": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#the-whole-game---status-update", - "href": "archive/2023-09-posit-conf/intro-03-what-makes-a-model.html#the-whole-game---status-update", + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-4", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-4", "title": "3 - What makes a model?", - "section": "The whole game - status update", - "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" - }, - { - "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#tuning-parameters", - "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#tuning-parameters", - "title": "5 - Tuning models", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "section": "To specify a model ", + "text": "To specify a model \n\nlinear_reg() %>%\n set_engine(\"stan\")\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: stan" }, { - "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters", - "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters", - "title": "5 - Tuning models", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-5", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-5", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters-1", - "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters-1", - "title": "5 - Tuning models", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-6", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-6", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" }, { - "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#specifying-tuning-parameters", - "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#specifying-tuning-parameters", - "title": "5 - Tuning models", - "section": "Specifying tuning parameters", - "text": "Specifying tuning parameters\nLet’s take our previous random forest workflow and tag for tuning the minimum number of data points in each node:\n\nrf_spec <- rand_forest(min_n = tune()) %>% \n set_mode(\"classification\")\n\nrf_wflow <- workflow(tip ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-7", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-7", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"regression\")\n#> Decision Tree Model Specification (regression)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#try-out-multiple-values", - "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#try-out-multiple-values", - "title": "5 - Tuning models", - "section": "Try out multiple values", - "text": "Try out multiple values\ntune_grid() works similar to fit_resamples() but covers multiple parameter values:\n\nset.seed(22)\nrf_res <- tune_grid(\n rf_wflow,\n taxi_folds,\n grid = 5\n)" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-8", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#to-specify-a-model-8", + "title": "3 - What makes a model?", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" }, { - "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#compare-results", - "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#compare-results", - "title": "5 - Tuning models", - "section": "Compare results", - "text": "Compare results\nInspecting results and selecting the best-performing hyperparameter(s):\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 33 roc_auc binary 0.623 10 0.0149 Preprocessor1_Model1\n#> 2 31 roc_auc binary 0.622 10 0.0154 Preprocessor1_Model3\n#> 3 21 roc_auc binary 0.620 10 0.0149 Preprocessor1_Model4\n#> 4 13 roc_auc binary 0.617 10 0.0137 Preprocessor1_Model5\n#> 5 6 roc_auc binary 0.611 10 0.0156 Preprocessor1_Model2\n\nbest_parameter <- select_best(rf_res)\nbest_parameter\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 33 Preprocessor1_Model1\n\ncollect_metrics() and autoplot() are also available." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-1", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-1", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code so it creates a different model.\n\n\n\n05:00\n\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#the-final-fit", - "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#the-final-fit", - "title": "5 - Tuning models", - "section": "The final fit", - "text": "The final fit\n\nrf_wflow <- finalize_workflow(rf_wflow, best_parameter)\n\nfinal_fit <- last_fit(rf_wflow, taxi_split) \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.913 Preprocessor1_Model1\n#> 2 roc_auc binary 0.648 Preprocessor1_Model1" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#models-well-be-using-today", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#models-well-be-using-today", + "title": "3 - What makes a model?", + "section": "Models we’ll be using today", + "text": "Models we’ll be using today\n\nLinear regression\nDecision trees" }, { - "objectID": "archive/2023-09-posit-conf/intro-05-tuning-models.html#your-turn", - "href": "archive/2023-09-posit-conf/intro-05-tuning-models.html#your-turn", - "title": "5 - Tuning models", - "section": "Your turn", - "text": "Your turn\n\nModify your model workflow to tune one or more parameters.\nUse grid search to find the best parameter(s).\n\n\n\n−+\n05:00" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression", + "title": "3 - What makes a model?", + "section": "Linear regression", + "text": "Linear regression" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#looking-at-the-predictors", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#looking-at-the-predictors", - "title": "Extras - Recipes", - "section": "Looking at the predictors", - "text": "Looking at the predictors\n\ntaxi_train\n#> # A tibble: 8,000 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 17.2 Chicago Independents no Thu Feb 16\n#> 2 yes 0.88 City Service yes Thu Mar 8\n#> 3 yes 18.1 other no Mon Feb 18\n#> 4 yes 12.2 Chicago Independents no Sun Mar 21\n#> 5 yes 0.94 Sun Taxi yes Sat Apr 23\n#> 6 yes 17.5 Flash Cab no Fri Mar 12\n#> 7 yes 17.7 other no Sun Jan 6\n#> 8 yes 1.85 Taxicab Insurance Agency Llc no Fri Apr 12\n#> 9 yes 0.53 Sun Taxi no Tue Mar 18\n#> 10 yes 6.65 Taxicab Insurance Agency Llc no Sun Apr 11\n#> # ℹ 7,990 more rows" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression-1", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression-1", + "title": "3 - What makes a model?", + "section": "Linear regression", + "text": "Linear regression" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#working-with-other-models", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#working-with-other-models", - "title": "Extras - Recipes", - "section": "Working with other models", - "text": "Working with other models\nSome models can’t handle non-numeric data\n\nLinear Regression\nK Nearest Neighbors\n\n\n\nSome models struggle if numeric predictors aren’t scaled\n\nK Nearest Neighbors\nAnything using gradient descent" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression-2", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#linear-regression-2", + "title": "3 - What makes a model?", + "section": "Linear regression", + "text": "Linear regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nOutcome modeled as linear combination of predictors:\n\n\\(\\mbox{latency} = \\beta_0 + \\beta_1\\cdot\\mbox{age} + \\epsilon\\)\n\nFind a line that minimizes the mean squared error (MSE)" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#types-of-needed-preprocessing", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#types-of-needed-preprocessing", - "title": "Extras - Recipes", - "section": "Types of needed preprocessing", - "text": "Types of needed preprocessing\n\nDo qualitative predictors require a numeric encoding?\nShould columns with a single unique value be removed?\nDoes the model struggle with missing data?\nDoes the model struggle with correlated predictors?\nShould predictors be centered and scaled?\nIs it helpful to transform predictors to be more symmetric?\n\n\nhttps://www.tmwr.org/pre-proc-table.html" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing", - "title": "Extras - Recipes", - "section": "Two types of preprocessing", - "text": "Two types of preprocessing" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees-1", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees-1", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing-1", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing-1", - "title": "Extras - Recipes", - "section": "Two types of preprocessing", - "text": "Two types of preprocessing" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees-2", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#decision-trees-2", + "title": "3 - What makes a model?", + "section": "Decision trees", + "text": "Decision trees" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#general-definitions", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#general-definitions", - "title": "Extras - Recipes", - "section": "General definitions", - "text": "General definitions\n\nData preprocessing are the steps that you take to make your model successful.\nFeature engineering are what you do to the original predictors to make the model do the least work to perform great." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", + "title": "3 - What makes a model?", + "section": "All models are wrong, but some are useful!", + "text": "All models are wrong, but some are useful!\n\n\nLinear regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#working-with-dates", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#working-with-dates", - "title": "Extras - Recipes", - "section": "Working with dates", - "text": "Working with dates\nDatetime variables are automatically converted to an integer if given as a raw predictor. To avoid this, it can be re-encoded as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nLeap year\nIndicators for holidays" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", + "title": "3 - What makes a model?", + "section": "Workflows bind preprocessors and models", + "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn", - "title": "Extras - Recipes", - "section": "Your turn", - "text": "Your turn\n\n\nWhat other transformations could we do with the raw time variable?\nRemember that the transformations are tied to the specific modeling problem.\n\n\n\n−+\n03:00" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#what-is-wrong-with-this", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#what-is-wrong-with-this", + "title": "3 - What makes a model?", + "section": "What is wrong with this?", + "text": "What is wrong with this?" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-transformations", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#two-types-of-transformations", - "title": "Extras - Recipes", - "section": "Two types of transformations", - "text": "Two types of transformations\n\n\n\nStatic\n\nSquare root, log, inverse\nDummies for known levels\nDate time extractions\n\n\nTrained\n\nCentering & scaling\nImputation\nPCA\nAnything for unknown factor levels\n\n\n\nTrained methods need to calculate sufficient information to be applied again." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#why-a-workflow", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#why-a-workflow", + "title": "3 - What makes a model?", + "section": "Why a workflow()? ", + "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering tomorrow!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#the-recipes-package", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#the-recipes-package", - "title": "Extras - Recipes", - "section": "The recipes package", - "text": "The recipes package\n\n\nModular + extensible\nWorks well with pipes ,|> and %>%\nDeferred evaluation\nIsolates test data from training data\nCan do things formulas can’t" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-1", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-1", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\ntree_spec %>% \n fit(latency ~ ., data = frog_train) \n#> parsnip model object\n#> \n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-2", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-2", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\nworkflow() %>%\n add_formula(latency ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = frog_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-1", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-1", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())\n\n\nStart by calling recipe() to denote the data source and variables used." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-3", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#a-model-workflow-3", + "title": "3 - What makes a model?", + "section": "A model workflow ", + "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\nworkflow(latency ~ ., tree_spec) %>% \n fit(data = frog_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 456 \n#> \n#> node), split, n, deviance, yval\n#> * denotes terminal node\n#> \n#> 1) root 456 2197966.00 92.90351 \n#> 2) age>=4.947975 256 252347.40 60.89844 \n#> 4) treatment=control 131 91424.06 48.42748 *\n#> 5) treatment=gentamicin 125 119197.90 73.96800 *\n#> 3) age< 4.947975 200 1347741.00 133.87000 \n#> 6) treatment=control 140 986790.70 118.25710 \n#> 12) reflex=mid,full 129 754363.70 111.56590 *\n#> 13) reflex=low 11 158918.20 196.72730 *\n#> 7) treatment=gentamicin 60 247194.60 170.30000 \n#> 14) age< 4.664439 30 102190.20 147.83330 \n#> 28) age>=4.566638 22 53953.86 129.77270 *\n#> 29) age< 4.566638 8 21326.00 197.50000 *\n#> 15) age>=4.664439 30 114719.40 192.76670 *" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-2", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-2", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())\n\n\nSpecify what actions to take by adding step_*()s." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-2", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-2", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code so it uses a linear model.\n\n\n\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-3", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-3", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%   step_normalize(all_numeric_predictors())\n\n\nUse {tidyselect} and recipes-specific selectors to denote affected variables." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#predict-with-your-model", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#predict-with-your-model", + "title": "3 - What makes a model?", + "section": "Predict with your model ", + "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"regression\")\n\ntree_fit <-\n workflow(latency ~ ., tree_spec) %>% \n fit(data = frog_train)" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe", - "title": "Extras - Recipes", - "section": "Using a recipe", - "text": "Using a recipe\n\ntaxi_rec <- recipe(tip ~ ., data = taxi_train) %>%\n  step_unknown(all_nominal_predictors()) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(distance, offset = 0.5) %>%   step_normalize(all_numeric_predictors())\n\n\nSave the recipe we like so that we can use it in various places, e.g., with different models." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-3", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-3", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = frog_test)\nWhat do you get?\n\n\n\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe-with-workflows", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe-with-workflows", - "title": "Extras - Recipes", - "section": "Using a recipe with workflows", - "text": "Using a recipe with workflows\nRecipes are typically combined with a model in a workflow() object:\n\n\ntaxi_wflow <- workflow() %>%\n  add_recipe(taxi_rec) %>%\n  add_model(linear_reg())" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-4", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-4", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = frog_test)\nWhat do you get?\n\n\n\n03:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#recipes-are-estimated", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#recipes-are-estimated", - "title": "Extras - Recipes", - "section": "Recipes are estimated", - "text": "Recipes are estimated\nEvery preprocessing step in a recipe that involved calculations uses the training set. For example:\n\nLevels of a factor\nDetermination of zero-variance\nNormalization\nFeature extraction\n\nOnce a recipe is added to a workflow, this occurs when fit() is called." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#debugging-a-recipe", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#debugging-a-recipe", - "title": "Extras - Recipes", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. taxi_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4.\n\n\n\n\nAnother function, bake(), is analogous to predict(), and gives you the processed data back." + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model-1", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model-1", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\nroundint = FALSE is only to quiet a warning" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn-1", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn-1", - "title": "Extras - Recipes", + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model-2", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#understand-your-model-2", + "title": "3 - What makes a model?", + "section": "Understand your model ", + "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" + }, + { + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-5", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-5", + "title": "3 - What makes a model?", "section": "Your turn", - "text": "Your turn\n\n\nTake the recipe and prep() then bake() it to see what the resulting data set looks like.\nTry removing steps to see how the result changes.\n\n\n\n\n−+\n05:00" + "text": "Your turn\n\nExtract the model engine object from your fitted linear workflow.\n⚠️ Never predict() with any extracted components!\n\n\n\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#printing-a-recipe", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#printing-a-recipe", - "title": "Extras - Recipes", - "section": "Printing a recipe", - "text": "Printing a recipe\n\ntaxi_rec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 6\n#> \n#> ── Operations\n#> • Unknown factor level assignment for: all_nominal_predictors()\n#> • Dummy variables from: all_nominal_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Log transformation on: distance\n#> • Centering and scaling for: all_numeric_predictors()" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#deploying-a-model", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#deploying-a-model", + "title": "3 - What makes a model?", + "section": "Deploying a model ", + "text": "Deploying a model \nHow do you use your new tree_fit model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"frog_hatching\")\nv\n#> \n#> ── frog_hatching ─ <butchered_workflow> model for deployment \n#> A rpart regression modeling workflow using 4 features\n\nLearn more at https://vetiver.rstudio.com" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#prepping-a-recipe", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#prepping-a-recipe", - "title": "Extras - Recipes", - "section": "Prepping a recipe", - "text": "Prepping a recipe\n\nprep(taxi_rec)\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 6\n#> \n#> ── Training information\n#> Training data contained 8000 data points and no incomplete rows.\n#> \n#> ── Operations\n#> • Unknown factor level assignment for: company and local, ... |\n#> Trained\n#> • Dummy variables from: company, local, dow, month | Trained\n#> • Zero variance filter removed: company_unknown, ... | Trained\n#> • Log transformation on: distance | Trained\n#> • Centering and scaling for: distance and hour, ... | Trained" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#deploy-your-model-1", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#deploy-your-model-1", + "title": "3 - What makes a model?", + "section": "Deploy your model ", + "text": "Deploy your model \nHow do you use your new model tree_fit in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 2 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/vetiver\n#> ├──/ping (GET)\n#> └──/predict (POST)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#baking-a-recipe", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#baking-a-recipe", - "title": "Extras - Recipes", - "section": "Baking a recipe", - "text": "Baking a recipe\n\nprep(taxi_rec) %>%\n bake(new_data = taxi_train)\n#> # A tibble: 8,000 × 19\n#> distance hour tip company_City.Service company_Flash.Cab company_Sun.Taxi company_Taxi.Affiliatio…¹ company_Taxicab.Insu…² company_other local_no dow_Mon dow_Tue dow_Wed dow_Thu dow_Fri dow_Sat\n#> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 1.38 0.418 yes -0.366 -0.333 -0.403 -0.450 -0.379 -0.609 0.484 -0.396 -0.441 -0.461 2.01 -0.428 -0.317\n#> 2 -0.729 -1.42 yes 2.73 -0.333 -0.403 -0.450 -0.379 -0.609 -2.07 -0.396 -0.441 -0.461 2.01 -0.428 -0.317\n#> 3 1.42 0.877 yes -0.366 -0.333 -0.403 -0.450 -0.379 1.64 0.484 2.53 -0.441 -0.461 -0.497 -0.428 -0.317\n#> 4 1.11 1.57 yes -0.366 -0.333 -0.403 -0.450 -0.379 -0.609 0.484 -0.396 -0.441 -0.461 -0.497 -0.428 -0.317\n#> 5 -0.694 2.03 yes -0.366 -0.333 2.48 -0.450 -0.379 -0.609 -2.07 -0.396 -0.441 -0.461 -0.497 -0.428 3.15 \n#> 6 1.39 -0.502 yes -0.366 3.01 -0.403 -0.450 -0.379 -0.609 0.484 -0.396 -0.441 -0.461 -0.497 2.34 -0.317\n#> 7 1.40 -1.88 yes -0.366 -0.333 -0.403 -0.450 -0.379 1.64 0.484 -0.396 -0.441 -0.461 -0.497 -0.428 -0.317\n#> 8 -0.289 -0.502 yes -0.366 -0.333 -0.403 -0.450 2.64 -0.609 0.484 -0.396 -0.441 -0.461 -0.497 2.34 -0.317\n#> 9 -0.971 0.877 yes -0.366 -0.333 2.48 -0.450 -0.379 -0.609 0.484 -0.396 2.27 -0.461 -0.497 -0.428 -0.317\n#> 10 0.631 -0.732 yes -0.366 -0.333 -0.403 -0.450 2.64 -0.609 0.484 -0.396 -0.441 -0.461 -0.497 -0.428 -0.317\n#> # ℹ 7,990 more rows\n#> # ℹ abbreviated names: ¹​company_Taxi.Affiliation.Services, ²​company_Taxicab.Insurance.Agency.Llc\n#> # ℹ 3 more variables: month_Feb <dbl>, month_Mar <dbl>, month_Apr <dbl>" + "objectID": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-6", + "href": "archive/2022-07-RStudio-conf/03-what-makes-a-model.html#your-turn-6", + "title": "3 - What makes a model?", + "section": "Your turn", + "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n05:00" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\nOnce a recipe as been estimated, there are various bits of information saved in it.\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#working-with-our-predictors", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#working-with-our-predictors", + "title": "5 - Feature engineering", + "section": "Working with our predictors", + "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for lm()).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn-2", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#your-turn-2", - "title": "Extras - Recipes", - "section": "Your turn", - "text": "Your turn\n\nTake a prepped recipe and use the tidy() function on it.\nUse the number argument to inspect different steps.\n\n\n\n\n−+\n05:00" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-is-feature-engineering", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-is-feature-engineering", + "title": "5 - Feature engineering", + "section": "What is feature engineering?", + "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPCA feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection." }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe-1", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe-1", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\n\nprep(taxi_rec) %>%\n tidy()\n#> # A tibble: 5 × 6\n#> number operation type trained skip id \n#> <int> <chr> <chr> <lgl> <lgl> <chr> \n#> 1 1 step unknown TRUE FALSE unknown_NTmu5 \n#> 2 2 step dummy TRUE FALSE dummy_cT3Uy \n#> 3 3 step zv TRUE FALSE zv_z22dk \n#> 4 4 step log TRUE FALSE log_QQ1iw \n#> 5 5 step normalize TRUE FALSE normalize_3cTJb" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#example-dates", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#example-dates", + "title": "5 - Feature engineering", + "section": "Example: Dates", + "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen a date column is used in its native format, it is usually converted by an R model to an integer.\n\n\nIt can be re-engineered as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe-2", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#tidying-a-recipe-2", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\n\nprep(taxi_rec) %>%\n tidy(number = 2)\n#> # A tibble: 20 × 3\n#> terms columns id \n#> <chr> <chr> <chr> \n#> 1 company City Service dummy_cT3Uy\n#> 2 company Flash Cab dummy_cT3Uy\n#> 3 company Sun Taxi dummy_cT3Uy\n#> 4 company Taxi Affiliation Services dummy_cT3Uy\n#> 5 company Taxicab Insurance Agency Llc dummy_cT3Uy\n#> 6 company other dummy_cT3Uy\n#> 7 company unknown dummy_cT3Uy\n#> 8 local no dummy_cT3Uy\n#> 9 local unknown dummy_cT3Uy\n#> 10 dow Mon dummy_cT3Uy\n#> 11 dow Tue dummy_cT3Uy\n#> 12 dow Wed dummy_cT3Uy\n#> 13 dow Thu dummy_cT3Uy\n#> 14 dow Fri dummy_cT3Uy\n#> 15 dow Sat dummy_cT3Uy\n#> 16 dow unknown dummy_cT3Uy\n#> 17 month Feb dummy_cT3Uy\n#> 18 month Mar dummy_cT3Uy\n#> 19 month Apr dummy_cT3Uy\n#> 20 month unknown dummy_cT3Uy" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#general-definitions", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#general-definitions", + "title": "5 - Feature engineering", + "section": "General definitions ", + "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\nIn a little bit, we’ll see successful (and unsuccessful) feature engineering methods for our example data.\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe-in-tidymodels", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#using-a-recipe-in-tidymodels", - "title": "Extras - Recipes", - "section": "Using a recipe in tidymodels", - "text": "Using a recipe in tidymodels\nThe recommended way to use a recipe in tidymodels is to use it as part of a workflow().\n\ntaxi_wflow <- workflow() %>% \n add_recipe(taxi_rec) %>% \n add_model(linear_reg())\n\nWhen used in this way, you don’t need to worry about prep() and bake() as it is handled for you." + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#the-nhl-data", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#the-nhl-data", + "title": "5 - Feature engineering", + "section": "The NHL data 🏒", + "text": "The NHL data 🏒\n\nFrom Pittsburgh Penguins games, 12,147 shots\nData from the 2015-2016 season\n\n\nLet’s predict whether a shot is on-goal (a goal or blocked by goaltender) or not." }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-recipes.html#more-information", - "href": "archive/2023-09-posit-conf/intro-extra-recipes.html#more-information", - "title": "Extras - Recipes", - "section": "More information", - "text": "More information\n\nhttps://recipes.tidymodels.org/\nhttps://www.tmwr.org/recipes.html" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#case-study", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#case-study", + "title": "5 - Feature engineering", + "section": "Case study", + "text": "Case study\n\nlibrary(tidymodels)\nlibrary(ongoal)\n\ntidymodels_prefer()\n\nglimpse(season_2015)\n#> Rows: 12,147\n#> Columns: 17\n#> $ on_goal <fct> yes, no, no, yes, no, no, yes, no, yes, no, no, no, …\n#> $ period <int> 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1…\n#> $ period_type <fct> regular, regular, regular, regular, regular, regular…\n#> $ coord_x <dbl> -53, 68, -42, -77, -67, 55, 77, 62, 59, 76, 44, 62, …\n#> $ coord_y <dbl> -18, -12, -18, 9, -5, -12, 13, 14, -5, -6, 7, -2, -2…\n#> $ game_time <dbl> 0.300000, 0.900000, 1.250000, 1.783333, 2.050000, 3.…\n#> $ strength <fct> even, even, even, even, even, even, even, even, even…\n#> $ player <fct> victor_hedman, evgeni_malkin, jason_garrison, ondrej…\n#> $ player_diff <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ offense_team <fct> TBL, PIT, TBL, TBL, TBL, PIT, PIT, PIT, PIT, PIT, PI…\n#> $ defense_team <fct> PIT, TBL, PIT, PIT, PIT, TBL, TBL, TBL, TBL, TBL, TB…\n#> $ offense_goal_diff <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ game_type <fct> playoff, playoff, playoff, playoff, playoff, playoff…\n#> $ position <fct> defenseman, center, defenseman, left_wing, defensema…\n#> $ dow <fct> Sat, Sat, Sat, Sat, Sat, Sat, Sat, Sat, Sat, Sat, Sa…\n#> $ month <fct> May, May, May, May, May, May, May, May, May, May, Ma…\n#> $ year <dbl> 2016, 2016, 2016, 2016, 2016, 2016, 2016, 2016, 2016…" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", - "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", - "title": "Extras - workflowsets", - "section": "How can we compare multiple model workflows at once?", - "text": "How can we compare multiple model workflows at once?" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#data-splitting-strategy", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#data-splitting-strategy", + "title": "5 - Feature engineering", + "section": "Data splitting strategy", + "text": "Data splitting strategy" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set", - "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set", - "title": "Extras - workflowsets", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec))\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#why-a-validation-set", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#why-a-validation-set", + "title": "5 - Feature engineering", + "section": "Why a validation set?", + "text": "Why a validation set?\nRecall that resampling gives us performance measures without using the test set.\nIt’s important to get good resampling statistics (e.g. \\(R^2\\)).\n\nThat usually means having enough data to estimate performance.\n\nWhen you have “a lot” of data, a validation set can be an efficient way to do this." }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", - "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", - "title": "Extras - workflowsets", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = taxi_folds)\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#splitting-the-nhl-data", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#splitting-the-nhl-data", + "title": "5 - Feature engineering", + "section": "Splitting the NHL data ", + "text": "Splitting the NHL data \n\nset.seed(23)\nnhl_split <- initial_split(season_2015, prop = 3/4)\nnhl_split\n#> <Training/Testing/Total>\n#> <9110/3037/12147>\n\nnhl_train_and_val <- training(nhl_split)\nnhl_test <- testing(nhl_split)\n\n## not testing\nnrow(nhl_train_and_val)\n#> [1] 9110\n \n## testing\nnrow(nhl_test)\n#> [1] 3037" }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", - "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", - "title": "Extras - workflowsets", - "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = taxi_folds) %>%\n rank_results()\n#> # A tibble: 4 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_decision… Prepro… accura… 0.915 0.00309 10 formula deci… 1\n#> 2 formula_decision… Prepro… roc_auc 0.624 0.0105 10 formula deci… 1\n#> 3 formula_rand_for… Prepro… accura… 0.924 0.00326 10 formula rand… 2\n#> 4 formula_rand_for… Prepro… roc_auc 0.615 0.0151 10 formula rand… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#validation-split", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#validation-split", + "title": "5 - Feature engineering", + "section": "Validation split ", + "text": "Validation split \nSince there are a lot of observations, we’ll use a validation set:\n\nset.seed(234)\nnhl_val <- validation_split(nhl_train_and_val, prop = 0.80)\nnhl_val\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7288/1822]> validation\n\n\nRemember that a validation split is a type of resample." }, { - "objectID": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#your-turn", - "href": "archive/2023-09-posit-conf/intro-extra-workflowsets.html#your-turn", - "title": "Extras - workflowsets", + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn", + "title": "5 - Feature engineering", "section": "Your turn", - "text": "Your turn\n\nWhen do you think a workflow set would be useful?\n\n\n\n−+\n03:00" + "text": "Your turn\n\nLet’s explore the training set data.\nUse the function plot_nhl_shots() for nice spatial plots of the data.\n\n\n\nnhl_train <- analysis(nhl_val$splits[[1]])\n\nset.seed(100)\nnhl_train %>% \n sample_n(200) %>%\n plot_nhl_shots(emphasis = position)\n\n\n\n\n\n\n\n\n\n\n\n\n08:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section", - "title": "2 - Tu presupuesto de datos", - "section": "", - "text": "“¡¡Que emoción!!”" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#prepare-your-data-for-modeling", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#prepare-your-data-for-modeling", + "title": "5 - Feature engineering", + "section": "Prepare your data for modeling ", + "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of feature engineering steps that provide preprocessing tools to be applied to data.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago", - "title": "2 - Tu presupuesto de datos", - "section": "Datos de viajes en taxi de Chicago", - "text": "Datos de viajes en taxi de Chicago\n\n\n\nLa ciudad de Chicago publica datos a nivel de viaje\nSacamos una muestra de 10,000 viajes que ocurrieron al principio del 2022\n\n\n\n\n\nhttps://www.svgrepo.com/svg/8322/taxi" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#a-first-recipe", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#a-first-recipe", + "title": "5 - Feature engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago-1", - "title": "2 - Tu presupuesto de datos", - "section": "Datos de viajes en taxi de Chicago", - "text": "Datos de viajes en taxi de Chicago\n\n\n\nN = 10,000\nUn resultado nominale, propina, con los niveles \"si\" y \"no\"\nVarias variables nominales, por ejemplo la identificación del Taxi, y el tipo de pago\nVarias variables numéricas, por ejemplo la distancia del viaje, y los subtotales del cargo por el viaje\n\n\n\n\n\nCredit: https://unsplash.com/photos/7_r85l4eht8\n\n\n“Fare subtotals” refers to the fare itself, tax, tolls, tip amount.\nActual variables in our data:\ntip: Whether the rider left a tip. A factor with levels “yes” and “no”.\ndistance: The trip distance, in odometer miles.\ncompany: The taxi company, as a factor. Companies that occurred few times were binned as “other”.\nlocal: Whether the trip started in the same community area as it began. See the source data for community area values.\ndow: The day of the week in which the trip began, as a factor.\nmonth: The month in which the trip began, as a factor.\nhour: The hour of the day in which the trip began, as a numeric." + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#a-first-recipe-1", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#a-first-recipe-1", + "title": "5 - Feature engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nsummary(nhl_rec)\n#> # A tibble: 17 × 4\n#> variable type role source \n#> <chr> <chr> <chr> <chr> \n#> 1 period numeric predictor original\n#> 2 period_type nominal predictor original\n#> 3 coord_x numeric predictor original\n#> 4 coord_y numeric predictor original\n#> 5 game_time numeric predictor original\n#> 6 strength nominal predictor original\n#> 7 player nominal predictor original\n#> 8 player_diff numeric predictor original\n#> 9 offense_team nominal predictor original\n#> 10 defense_team nominal predictor original\n#> 11 offense_goal_diff numeric predictor original\n#> 12 game_type nominal predictor original\n#> 13 position nominal predictor original\n#> 14 dow nominal predictor original\n#> 15 month nominal predictor original\n#> 16 year numeric predictor original\n#> 17 on_goal nominal outcome original" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#se-pueden-usar-estas-variables", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#se-pueden-usar-estas-variables", - "title": "2 - Tu presupuesto de datos", - "section": "¿Se pueden usar estas variables?", - "text": "¿Se pueden usar estas variables?\n\n¿Será ético, o hasta legal, utilizar esta variable?\n¿La variable estará disponible al momento de predecir?\n¿La variable contribuye a la explicación de los resultados?" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#create-indicator-variables", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#create-indicator-variables", + "title": "5 - Feature engineering", + "section": "Create indicator variables ", + "text": "Create indicator variables \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-viajes-en-taxi-de-chicago-2", - "title": "2 - Tu presupuesto de datos", - "section": "Datos de viajes en taxi de Chicago", - "text": "Datos de viajes en taxi de Chicago\nBaja los datos, y cargarlos en tu sesión de R\n\ndownload.file(\"https://github.com/edgararuiz/tidymodels-workshops/blob/main/archive/2024-03-conectaR-spanish/taxi.rds\",\n \"taxi.rds\")\n\ntaxi <- readRDS(taxi.rds)\n\n\ntaxi\n#> # A tibble: 10,000 × 7\n#> propina distancia compania local dia mes hora\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 si 17.2 Chicago Independents no Jue Feb 16\n#> 2 si 0.88 City Service si Jue Mar 8\n#> 3 si 18.1 otra no Lun Feb 18\n#> 4 si 20.7 Chicago Independents no Lun Abr 8\n#> 5 si 12.2 Chicago Independents no Dom Mar 21\n#> 6 si 0.94 Sun Taxi si Sab Abr 23\n#> 7 si 17.5 Flash Cab no Vie Mar 12\n#> 8 si 17.7 otra no Dom Ene 6\n#> 9 si 1.85 Taxicab Insurance Agency Llc no Vie Abr 12\n#> 10 si 1.47 City Service no Mar Mar 14\n#> # ℹ 9,990 more rows" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#filter-out-constant-columns", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#filter-out-constant-columns", + "title": "5 - Feature engineering", + "section": "Filter out constant columns ", + "text": "Filter out constant columns \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos", - "title": "2 - Tu presupuesto de datos", - "section": "Separando y “gastando” datos", - "text": "Separando y “gastando” datos\nPara el aprendizaje automatico, separamos los datos, unos para entranamiento, y los otros para la “prueba final”:\n\n\nLos datos de entrenamiento se utilizan para estimar los parametros del modelo\nLos datos de prueba se ponen aparte para medir la calidad del modelo\n\n\n\n🚫 No utilize los datos de prueba durante el entrenamiento 🚫" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#normalization", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#normalization", + "title": "5 - Feature engineering", + "section": "Normalization ", + "text": "Normalization \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-1", - "title": "2 - Tu presupuesto de datos", - "section": "Separando y “gastando” datos", - "text": "Separando y “gastando” datos" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#reduce-correlation", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#reduce-correlation", + "title": "5 - Feature engineering", + "section": "Reduce correlation ", + "text": "Reduce correlation \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-2", - "title": "2 - Tu presupuesto de datos", - "section": "Separando y “gastando” datos", - "text": "Separando y “gastando” datos\n\nGastando mucho datos en el entrenamiento previene que tengamos una buena manera de medir la calidad del modelo\n\n\n\nGastando mucho datos en prueba previente que tegamos una buena estimación de los parametros" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps", + "title": "5 - Feature engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno", - "title": "2 - Tu presupuesto de datos", - "section": "Tu turno", - "text": "Tu turno\n\n¿Cuando deberiamos separar nuestros datos?\n\n\n\n−+\n03:00" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps-1", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps-1", + "title": "5 - Feature engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = on_goal)\n\n\nA fancy machine learning supervised dimension reduction technique…\n\nNote that this uses the outcome, and it is from an extension package" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#la-separación-inicial", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#la-separación-inicial", - "title": "2 - Tu presupuesto de datos", - "section": "La separación inicial ", - "text": "La separación inicial \n\nset.seed(123)\ntaxi_separar <- initial_split(taxi)\ntaxi_separar\n#> <Training/Testing/Total>\n#> <7500/2500/10000>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps-2", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#other-possible-steps-2", + "title": "5 - Feature engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nnhl_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_ns(coord_y, coord_x, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#para-que-es-set.seed", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#para-que-es-set.seed", - "title": "2 - Tu presupuesto de datos", - "section": "¿Para que es set.seed()?", - "text": "¿Para que es set.seed()?\n\nPara el propósito de separar datos, R puede generar números “seudo-aleatorios”, que aunque parezcan comportarse como números aleatorios, en realidad se generan de una manera específica basada en una “semilla” (seed)\n\n\nSi utilizamos la misma semilla nos permite reproducir los mismos resultados.\n\n\nEl número en sí que elijes realmente no importa. Lo que si importa, es que no experimentes con diferentes números de la semilla con el proposito de mejorar la calidad del modelo." + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-1", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-1", + "title": "5 - Feature engineering", + "section": "Your turn", + "text": "Your turn\n\nCreate a recipe() for the on-goal data to :\n\ncreate one-hot indicator variables\nremove zero-variance variables\n\n\n\n\n03:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#accediendo-los-datos", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#accediendo-los-datos", - "title": "2 - Tu presupuesto de datos", - "section": "Accediendo los datos ", - "text": "Accediendo los datos \n\ntaxi_entrenar <- training(taxi_separar)\ntaxi_test <- testing(taxi_separar)" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#minimal-recipe", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#minimal-recipe", + "title": "5 - Feature engineering", + "section": "Minimal recipe ", + "text": "Minimal recipe \n\nnhl_indicators <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#el-set-the-entrenamiento", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#el-set-the-entrenamiento", - "title": "2 - Tu presupuesto de datos", - "section": "El set the entrenamiento ", - "text": "El set the entrenamiento \n\ntaxi_entrenar\n#> # A tibble: 7,500 × 7\n#> propina distancia compania local dia mes hora\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 si 0.7 Taxi Affiliation Services si Mar Mar 18\n#> 2 si 0.99 Sun Taxi si Mar Ene 8\n#> 3 si 1.78 otra no Sab Mar 22\n#> 4 si 0 Taxi Affiliation Services si Mie Abr 15\n#> 5 si 0 Taxi Affiliation Services no Dom Ene 21\n#> 6 si 2.3 otra no Sab Abr 21\n#> 7 si 6.35 Sun Taxi no Mie Mar 16\n#> 8 si 2.79 otra no Dom Feb 14\n#> 9 si 16.6 otra no Dom Abr 18\n#> 10 si 0.02 Chicago Independents si Dom Abr 15\n#> # ℹ 7,490 more rows" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#using-a-workflow", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#using-a-workflow", + "title": "5 - Feature engineering", + "section": "Using a workflow ", + "text": "Using a workflow \n\nset.seed(9)\n\nnhl_glm_wflow <-\n workflow() %>%\n add_recipe(nhl_indicators) %>%\n add_model(logistic_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nnhl_glm_res <-\n nhl_glm_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_glm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.555 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.558 1 NA Preprocessor1_Model1" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#el-set-de-prueba", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#el-set-de-prueba", - "title": "2 - Tu presupuesto de datos", - "section": "El set de prueba ", - "text": "El set de prueba \n🙈\n\nEn el set de prueba, hay 2500 entradas, con 7 columnas" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-2", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-2", + "title": "5 - Feature engineering", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n05:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno-1", - "title": "2 - Tu presupuesto de datos", - "section": "Tu turno", - "text": "Tu turno\n\nSepara tus datos, el 20% tiene que ser para prueba\nTrata diferentes valores para set.seed() para ver como cambian los resultados\n\n\n\n−+\n05:00" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#holdout-predictions", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#holdout-predictions", + "title": "5 - Feature engineering", + "section": "Holdout predictions ", + "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nglm_val_pred <- collect_predictions(nhl_glm_res)\nglm_val_pred %>% slice(1:7)\n#> # A tibble: 7 × 7\n#> id .pred_yes .pred_no .row .pred_class on_goal .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 validation 0.198 8.02e- 1 10 no no Preprocessor1_Model1\n#> 2 validation 0.264 7.36e- 1 17 no yes Preprocessor1_Model1\n#> 3 validation 0.189 8.11e- 1 23 no no Preprocessor1_Model1\n#> 4 validation 1.00 8.39e-11 40 yes yes Preprocessor1_Model1\n#> 5 validation 0.322 6.78e- 1 41 no yes Preprocessor1_Model1\n#> 6 validation 1.00 8.39e-11 46 yes no Preprocessor1_Model1\n#> 7 validation 0.354 6.46e- 1 55 no no Preprocessor1_Model1" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#separando-y-gastando-datos-3", - "title": "2 - Tu presupuesto de datos", - "section": "Separando y “gastando” datos ", - "text": "Separando y “gastando” datos \n\nset.seed(123)\ntaxi_separar <- initial_split(taxi, prop = 0.8)\ntaxi_entrenar <- training(taxi_separar)\ntaxi_test <- testing(taxi_separar)\n\nnrow(taxi_entrenar)\n#> [1] 8000\nnrow(taxi_test)\n#> [1] 2000" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#two-class-data-1", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#two-class-data-1", + "title": "5 - Feature engineering", + "section": "Two class data", + "text": "Two class data\nThese definitions assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-1", - "title": "2 - Tu presupuesto de datos", - "section": "", - "text": "flowchart TD\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000 \n vl[Validación]\n style vl fill:#E5E7FD,stroke:#666,color:#000 \n ad --> tr\n ad --> ts\n ad --> vl" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curves", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curves", + "title": "5 - Feature engineering", + "section": "ROC curves", + "text": "ROC curves\nTo make an ROC (receiver operator characteristic) curve, we:\n\ncalculate the sensitivity and specificity for all possible thresholds\nplot false positive rate (x-axis) versus true positive rate (y-axis)\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-validación", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#datos-de-validación", - "title": "2 - Tu presupuesto de datos", - "section": "Datos de validación", - "text": "Datos de validación\n\nset.seed(123)\ninitial_validation_split(taxi, prop = c(0.6, 0.2))\n#> <Training/Validation/Testing/Total>\n#> <6000/2000/2000/10000>" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curves-1", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curves-1", + "title": "5 - Feature engineering", + "section": "ROC curves ", + "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\nroc_curve_points <- glm_val_pred %>% roc_curve(truth = on_goal, estimate = .pred_yes)\nroc_curve_points %>% slice(1, 50, 100)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.139 0.0303 0.977\n#> 3 0.272 0.0642 0.955\n\nglm_val_pred %>% roc_auc(truth = on_goal, estimate = .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.558" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#tu-turno-2", - "title": "2 - Tu presupuesto de datos", - "section": "Tu turno", - "text": "Tu turno\n\nExplora los datos en taxi_entrenar por tí mismo\n\n¿Cual es la distribución de los resultados de propina?\n¿Cual es la distribución de las variables numéricas, por ejemplo distancia?\n¿Como cambian los resultados de propina a través de las variables categóricas?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curve-plot", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#roc-curve-plot", + "title": "5 - Feature engineering", + "section": "ROC curve plot ", + "text": "ROC curve plot \n\nautoplot(roc_curve_points)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-2", - "title": "2 - Tu presupuesto de datos", - "section": "", - "text": "taxi_entrenar %>% \n ggplot(aes(x = propina)) +\n geom_bar()" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-3", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-3", + "title": "5 - Feature engineering", + "section": "Your turn", + "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\n\n\n\n05:00" + }, + { + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-4", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-4", + "title": "5 - Feature engineering", + "section": "Your turn", + "text": "Your turn\n\nWhat data are being used for this ROC curve plot?\n\n\n\n03:00" + }, + { + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-do-we-do-with-the-player-data", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-do-we-do-with-the-player-data", + "title": "5 - Feature engineering", + "section": "What do we do with the player data? 🏒", + "text": "What do we do with the player data? 🏒\nThere are 597 unique player values in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the player column with the estimated effect of that predictor\n\n\n\nLet’s use an effect encoding." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-3", - "title": "2 - Tu presupuesto de datos", - "section": "", - "text": "taxi_entrenar %>% \n ggplot(aes(x = propina, fill = local)) +\n geom_bar() +\n scale_fill_viridis_d(end = .5)" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-is-an-effect-encoding", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#what-is-an-effect-encoding", + "title": "5 - Feature engineering", + "section": "What is an effect encoding?", + "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> on_goal player .row\n#> <fct> <fct> <int>\n#> 1 yes brian_dumoulin 1\n#> 2 yes patric_hornqvist 2\n#> 3 yes nikita_nesterov 3\n#> 4 yes jack_eichel 4\n#> 5 yes justin_williams 5\n#> 6 yes seth_jones 6\n#> 7 yes kris_letang 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> on_goal player .row\n#> <fct> <dbl> <int>\n#> 1 yes -0.114 1\n#> 2 yes 0.631 2\n#> 3 yes 0.142 3\n#> 4 yes 0.220 4\n#> 5 yes 0.248 5\n#> 6 yes 0.0733 6\n#> 7 yes 0.0774 7\n\n\nThe player column is replaced with the log-odds of being on goal.\n\nAs a reminder:\n\\[\\text{log-odds} = log\\left(\\frac{\\hat{p}}{1 - \\hat{p}}\\right)\\]\nwhere \\(\\hat{p}\\) is the on goal rate estimate.\nFor logistic regression, this is what the predictors are modeling. The log-odds are more likely to be linear with the outcome." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-4", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-4", - "title": "2 - Tu presupuesto de datos", - "section": "", - "text": "taxi_entrenar %>% \n ggplot(aes(x = hora, fill = propina)) +\n geom_bar()" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#per-player-statistics", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#per-player-statistics", + "title": "5 - Feature engineering", + "section": "Per-player statistics", + "text": "Per-player statistics\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these rates use partial pooling.\nPooling borrows strength across players and shrinks extreme values (e.g. zero or one) towards the mean for players with very few shots.\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for players with fewer shots by shrinking the estimate to the overall on-goal rate (55.2%)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-5", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-5", - "title": "2 - Tu presupuesto de datos", - "section": "", - "text": "taxi_entrenar %>% \n ggplot(aes(x = hora, fill = propina)) +\n geom_bar(position = \"fill\")" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#partial-pooling", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#partial-pooling", + "title": "5 - Feature engineering", + "section": "Partial pooling", + "text": "Partial pooling" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-6", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-6", - "title": "2 - Tu presupuesto de datos", - "section": "", - "text": "taxi_entrenar %>% \n ggplot(aes(x = distancia)) +\n geom_histogram(bins = 100) +\n facet_grid(vars(propina))" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#player-effects", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#player-effects", + "title": "5 - Feature engineering", + "section": "Player effects ", + "text": "Player effects \n\nlibrary(embed)\n\nnhl_effect_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_lencode_mixed(player, outcome = vars(on_goal)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-7", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#section-7", - "title": "2 - Tu presupuesto de datos", - "section": "", - "text": "Estratificar la muestra separará los datos dentro los valores de respuesta\n\nBased on our EDA, we know that the source data contains fewer \"no\" tip values than \"yes\". We want to make sure we allot equal proportions of those responses so that both the training and testing data have enough of each to give accurate estimates." + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#recipes-are-estimated", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#recipes-are-estimated", + "title": "5 - Feature engineering", + "section": "Recipes are estimated ", + "text": "Recipes are estimated \nPreprocessing steps in a recipe use the training set to compute quantities.\n\nWhat kind of quantities are computed for preprocessing?\n\nLevels of a factor\nWhether a column has zero variance\nNormalization\nFeature extraction\nEffect encodings\n\n\n\nWhen a recipe is part of a workflow, this estimation occurs when fit() is called." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#estratificar", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#estratificar", - "title": "2 - Tu presupuesto de datos", - "section": "Estratificar", - "text": "Estratificar\nUse strata = propina\n\nset.seed(123)\ntaxi_separar <- initial_split(taxi, prop = 0.8, strata = propina)\ntaxi_separar\n#> <Training/Testing/Total>\n#> <8000/2000/10000>" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#effect-encoding-results", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#effect-encoding-results", + "title": "5 - Feature engineering", + "section": "Effect encoding results ", + "text": "Effect encoding results \n\nnhl_effect_wflow <-\n nhl_glm_wflow %>%\n update_recipe(nhl_effect_rec)\n\nnhl_effect_res <-\n nhl_effect_wflow %>%\n fit_resamples(nhl_val, control = ctrl)\n\ncollect_metrics(nhl_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.540 1 NA Preprocessor1_Model1\n#> 2 roc_auc binary 0.551 1 NA Preprocessor1_Model1" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#estratificar-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#estratificar-1", - "title": "2 - Tu presupuesto de datos", - "section": "Estratificar", - "text": "Estratificar\nEstratificar la muestra usualmente ayuda, y con pocas consecuencias" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#angle", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#angle", + "title": "5 - Feature engineering", + "section": "Angle", + "text": "Angle\n\nnhl_angle_rec <-\n nhl_indicators %>%\n step_mutate(\n angle = abs(atan2(abs(coord_y), (89 - abs(coord_x))) * (180 / pi))\n )\n\n\n\nNote the danger of using step_mutate() – easy to have data leakage\ncoord_x is “distance from goal”. We subtract it from 89 to get the distance from the center of the ice. The abs() calls account for the fact that the goals might be on either side of (0, 0). The rest of it is the formula for going from (x, y) to angle in degrees." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#expectativas-del-taller---donde-estamos", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-02-presupuesto-de-datos.html#expectativas-del-taller---donde-estamos", - "title": "2 - Tu presupuesto de datos", - "section": "Expectativas del taller - Donde estamos", - "text": "Expectativas del taller - Donde estamos\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#fff,stroke:#eee,color:#ddd\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#fff,stroke:#eee,color:#ddd\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#fff,stroke:#eee,color:#ddd\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#fff,stroke:#eee,color:#ddd\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#fff,stroke:#eee,color:#ddd\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#fff,stroke:#eee,color:#ddd\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#fff,stroke:#eee,color:#ddd\n fm --> vm\n ts --> vm" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#distance", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#distance", + "title": "5 - Feature engineering", + "section": "Distance", + "text": "Distance\n\nnhl_distance_rec <-\n nhl_angle_rec %>%\n step_mutate(\n distance = sqrt((89 - abs(coord_x))^2 + abs(coord_y)^2),\n distance = log(distance)\n )" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#viendo-las-predicciones", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#viendo-las-predicciones", - "title": "4 - Evaluar modelos", - "section": "Viendo las predicciones", - "text": "Viendo las predicciones\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n relocate(propina, .pred_class, .pred_si, .pred_no)\n#> # A tibble: 8,000 × 10\n#> propina .pred_class .pred_si .pred_no distancia compania local dia mes \n#> <fct> <fct> <dbl> <dbl> <dbl> <fct> <fct> <fct> <fct>\n#> 1 si si 0.967 0.0333 17.2 Chicago In… no Jue Feb \n#> 2 si si 0.935 0.0646 0.88 City Servi… si Jue Mar \n#> 3 si si 0.967 0.0333 18.1 otra no Lun Feb \n#> 4 si si 0.949 0.0507 12.2 Chicago In… no Dom Mar \n#> 5 si si 0.821 0.179 0.94 Sun Taxi si Sab Abr \n#> 6 si si 0.967 0.0333 17.5 Flash Cab no Vie Mar \n#> 7 si si 0.967 0.0333 17.7 otra no Dom Ene \n#> 8 si si 0.938 0.0616 1.85 Taxicab In… no Vie Abr \n#> 9 si si 0.938 0.0616 0.53 Sun Taxi no Mar Mar \n#> 10 si si 0.931 0.0694 6.65 Taxicab In… no Dom Abr \n#> # ℹ 7,990 more rows\n#> # ℹ 1 more variable: hora <int>" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#behind-goal-line", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#behind-goal-line", + "title": "5 - Feature engineering", + "section": "Behind goal line", + "text": "Behind goal line\n\nnhl_behind_rec <-\n nhl_distance_rec %>%\n step_mutate(\n behind_goal_line = ifelse(abs(coord_x) >= 89, 1, 0)\n )" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión", - "title": "4 - Evaluar modelos", - "section": "Matriz de confusión ", - "text": "Matriz de confusión" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#fit-different-recipes", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#fit-different-recipes", + "title": "5 - Feature engineering", + "section": "Fit different recipes ", + "text": "Fit different recipes \nA workflow set can cross models and/or preprocessors and then resample them en masse.\n\nset.seed(9)\n\nnhl_glm_set_res <-\n workflow_set(\n list(`1_dummy` = nhl_indicators, `2_angle` = nhl_angle_rec, \n `3_dist` = nhl_distance_rec, `4_bgl` = nhl_behind_rec),\n list(logistic = logistic_reg())\n ) %>%\n workflow_map(fn = \"fit_resamples\", resamples = nhl_val, verbose = TRUE, control = ctrl)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión-1", - "title": "4 - Evaluar modelos", - "section": "Matriz de confusión ", - "text": "Matriz de confusión \n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n conf_mat(truth = propina, estimate = .pred_class)\n#> Truth\n#> Prediction si no\n#> si 7341 536\n#> no 43 80" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-5", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#your-turn-5", + "title": "5 - Feature engineering", + "section": "Your turn", + "text": "Your turn\n\nCreate a workflow set with 2 or 3 recipes.\n(Consider using recipes we’ve already created.)\nUse workflow_map() to resample the workflow set.\n\n\n\n08:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#matriz-de-confusión-2", - "title": "4 - Evaluar modelos", - "section": "Matriz de confusión ", - "text": "Matriz de confusión \n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n conf_mat(truth = propina, estimate = .pred_class) %>%\n autoplot(type = \"heatmap\")" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#compare-recipes", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#compare-recipes", + "title": "5 - Feature engineering", + "section": "Compare recipes", + "text": "Compare recipes\n\nlibrary(forcats)\ncollect_metrics(nhl_glm_set_res) %>%\n filter(.metric == \"roc_auc\") %>%\n mutate(\n features = gsub(\"_logistic\", \"\", wflow_id), \n features = fct_reorder(features, mean)\n ) %>%\n ggplot(aes(x = mean, y = features)) +\n geom_point(size = 3) +\n labs(y = NULL, x = \"ROC AUC (validation set)\")" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo", - "title": "4 - Evaluar modelos", - "section": "Mediciones de la calidad del modelo ", - "text": "Mediciones de la calidad del modelo \n\n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n accuracy(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#compare-recipes-1", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#compare-recipes-1", + "title": "5 - Feature engineering", + "section": "Compare recipes", + "text": "Compare recipes" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#el-riesgo-de-concentrarse-en-la-exactitud", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#el-riesgo-de-concentrarse-en-la-exactitud", - "title": "4 - Evaluar modelos", - "section": "El riesgo de concentrarse en la exactitud ", - "text": "El riesgo de concentrarse en la exactitud \nHay que tener cuidado utilizando exactitud (accuracy()) ya que nos puede dar “buenos” resultado se predecimos con datos que no están balanceados\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n mutate(.pred_class = factor(\"si\", levels = c(\"si\", \"no\"))) %>%\n accuracy(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.923" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#debugging-a-recipe", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#debugging-a-recipe", + "title": "5 - Feature engineering", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. encoded_players) can be estimated manually with a function called prep(). It is analogous to fit().\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-1", - "title": "4 - Evaluar modelos", - "section": "Mediciones de la calidad del modelo ", - "text": "Mediciones de la calidad del modelo \n\n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n sensitivity(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.994" + "objectID": "archive/2022-07-RStudio-conf/05-feature-engineering.html#more-on-recipes", + "href": "archive/2022-07-RStudio-conf/05-feature-engineering.html#more-on-recipes", + "title": "5 - Feature engineering", + "section": "More on recipes", + "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-2", - "title": "4 - Evaluar modelos", - "section": "Mediciones de la calidad del modelo ", - "text": "Mediciones de la calidad del modelo \n\n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n sensitivity(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.994\n\n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n specificity(truth = propina, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.130" + "objectID": "archive/2022-07-RStudio-conf/07-wrapping-up.html#your-turn", + "href": "archive/2022-07-RStudio-conf/07-wrapping-up.html#your-turn", + "title": "7 - Wrapping up", + "section": "Your turn", + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n05:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-3", - "title": "4 - Evaluar modelos", - "section": "Mediciones de la calidad del modelo ", - "text": "Mediciones de la calidad del modelo \nPara combinar multiples cálculos en una tabla, usa metric_set()\n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n taxi_metrics(truth = propina, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n#> 2 specificity binary 0.130\n#> 3 sensitivity binary 0.994" + "objectID": "archive/2022-07-RStudio-conf/07-wrapping-up.html#resources-to-keep-learning", + "href": "archive/2022-07-RStudio-conf/07-wrapping-up.html#resources-to-keep-learning", + "title": "7 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Twitter and at the tidyverse blog for updates!" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-4", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#mediciones-de-la-calidad-del-modelo-4", - "title": "4 - Evaluar modelos", - "section": "Mediciones de la calidad del modelo ", - "text": "Mediciones de la calidad del modelo \n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>%\n group_by(local) %>%\n taxi_metrics(truth = propina, estimate = .pred_class)\n#> # A tibble: 6 × 4\n#> local .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 si accuracy binary 0.898\n#> 2 no accuracy binary 0.935\n#> 3 si specificity binary 0.169\n#> 4 no specificity binary 0.116\n#> 5 si sensitivity binary 0.987\n#> 6 no sensitivity binary 0.996" + "objectID": "archive/2022-07-RStudio-conf/index.html", + "href": "archive/2022-07-RStudio-conf/index.html", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels presented at the 2022 RStudio conference. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#resultados-de-dos-clases", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#resultados-de-dos-clases", - "title": "4 - Evaluar modelos", - "section": "Resultados de dos clases", - "text": "Resultados de dos clases\nEstas métricas asumen que sabemos cual es límite para convertir probabilidades de predicción “suaves” a prediciones de clase “duras” . . .\n¿Es bueno un límite de 50%?\n¿Que pasaría si lo cambiamos a 80%?\n\nsensibilidad ⬇️, especificidad ⬆️\n\n\n¿Y si lo cambiamos a 20%?\n\nsensibilidad ⬆️, especificidad ⬇️" + "objectID": "archive/2022-07-RStudio-conf/index.html#welcome", + "href": "archive/2022-07-RStudio-conf/index.html#welcome", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels presented at the 2022 RStudio conference. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#varying-the-threshold", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#varying-the-threshold", - "title": "4 - Evaluar modelos", - "section": "Varying the threshold", - "text": "Varying the threshold" + "objectID": "archive/2022-07-RStudio-conf/index.html#is-this-workshop-for-me", + "href": "archive/2022-07-RStudio-conf/index.html#is-this-workshop-for-me", + "title": "Machine learning with tidymodels", + "section": "Is this workshop for me? ", + "text": "Is this workshop for me? \nThis course assumes intermediate R knowledge. This workshop is for you if:\n\nYou can use the magrittr pipe %>% and/or native pipe |>\nYou are familiar with functions from dplyr, tidyr, and ggplot2\nYou can read data into R, transform and reshape data, and make a wide variety of graphs\n\nWe expect participants to have some exposure to basic statistical concepts, but NOT intermediate or expert familiarity with modeling or machine learning." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#curvas-roc", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#curvas-roc", - "title": "4 - Evaluar modelos", - "section": "Curvas ROC", - "text": "Curvas ROC\nPara crear una “curva ROC”, osea una curva de característica operativa del receptor hacemos lo siguiente:\n\nCalcular la sensibilidad y especificidad de todos los límites posibles\nGráficar los falsos positivos en el axis X, contra los positivos verdaderos en el axis Y.\n\nYa que la sensibilidad es la proporción de positivos verdaderos, y la especificidad es la de los negativos verdaderos, entonces 1 - especificidad es la proporción de los falsos positivos.\n\nPodemos usar el area debajo de la curva (AUC = area under de curve) ROC como una métrica de clasificación:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." + "objectID": "archive/2022-07-RStudio-conf/index.html#preparation", + "href": "archive/2022-07-RStudio-conf/index.html#preparation", + "title": "Machine learning with tidymodels", + "section": "Preparation", + "text": "Preparation\nPlease join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2022.02), available at https://www.rstudio.com/download\nThe following R packages, which you can install from the R console:\n\n\ninstall.packages(c(\"DALEXtra\", \"doParallel\", \"embed\", \"forcats\",\n \"lme4\", \"ranger\", \"remotes\", \"rpart\", \n \"rpart.plot\", \"stacks\", \"tidymodels\",\n \"vetiver\", \"xgboost\"))\n\nremotes::install_github(\"topepo/ongoal@v0.0.2\")" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#curvas-roc-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#curvas-roc-1", - "title": "4 - Evaluar modelos", - "section": "Curvas ROC ", - "text": "Curvas ROC \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(taxi_ajustado, new_data = taxi_entrenar) %>% \n roc_curve(truth = propina, .pred_si) %>%\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.783 0.209 0.981 \n#> 3 1 1 0.00135\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>% \n roc_auc(truth = propina, .pred_si)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.691" + "objectID": "archive/2022-07-RStudio-conf/index.html#slides", + "href": "archive/2022-07-RStudio-conf/index.html#slides", + "title": "Machine learning with tidymodels", + "section": "Slides", + "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. There are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nDay One\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n\n\n\nDay Two\n\n05: Feature engineering\n06: Tuning hyperparameters\n07: Wrapping up\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#gráfica-de-curvas-roc", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#gráfica-de-curvas-roc", - "title": "4 - Evaluar modelos", - "section": "Gráfica de curvas ROC ", - "text": "Gráfica de curvas ROC \n\n\naugment(taxi_ajustado, new_data = taxi_entrenar) %>% \n roc_curve(truth = propina, .pred_si) %>%\n autoplot()" + "objectID": "archive/2022-07-RStudio-conf/index.html#code", + "href": "archive/2022-07-RStudio-conf/index.html#code", + "title": "Machine learning with tidymodels", + "section": "Code", + "text": "Code\nQuarto files for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno", - "title": "4 - Evaluar modelos", - "section": "Tu turno", - "text": "Tu turno\n\nCalcule y gráfique una curva ROC con su modelo\n¿Cuales son los datos que se utilizaron para esta curva ROC?\n\n\n\n−+\n05:00" + "objectID": "archive/2022-07-RStudio-conf/index.html#acknowledgments", + "href": "archive/2022-07-RStudio-conf/index.html#acknowledgments", + "title": "Machine learning with tidymodels", + "section": "Acknowledgments", + "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-1", - "title": "4 - Evaluar modelos", - "section": "Los peligros del sobreajustar ⚠️", - "text": "Los peligros del sobreajustar ⚠️" + "objectID": "archive/2022-07-RStudio-conf/index.html#reuse-and-licensing", + "href": "archive/2022-07-RStudio-conf/index.html#reuse-and-licensing", + "title": "Machine learning with tidymodels", + "section": "Reuse and licensing", + "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-2", - "title": "4 - Evaluar modelos", - "section": "Los peligros del sobreajustar ⚠️", - "text": "Los peligros del sobreajustar ⚠️" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching", + "title": "2 - Your data budget", + "section": "Data on tree frog hatching", + "text": "Data on tree frog hatching" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-3", - "title": "4 - Evaluar modelos", - "section": "Los peligros del sobreajustar ⚠️ ", - "text": "Los peligros del sobreajustar ⚠️ \n\ntaxi_ajustado %>%\n augment(taxi_entrenar)\n#> # A tibble: 8,000 × 10\n#> .pred_class .pred_si .pred_no propina distancia compania local dia mes \n#> <fct> <dbl> <dbl> <fct> <dbl> <fct> <fct> <fct> <fct>\n#> 1 si 0.967 0.0333 si 17.2 Chicago In… no Jue Feb \n#> 2 si 0.935 0.0646 si 0.88 City Servi… si Jue Mar \n#> 3 si 0.967 0.0333 si 18.1 otra no Lun Feb \n#> 4 si 0.949 0.0507 si 12.2 Chicago In… no Dom Mar \n#> 5 si 0.821 0.179 si 0.94 Sun Taxi si Sab Abr \n#> 6 si 0.967 0.0333 si 17.5 Flash Cab no Vie Mar \n#> 7 si 0.967 0.0333 si 17.7 otra no Dom Ene \n#> 8 si 0.938 0.0616 si 1.85 Taxicab In… no Vie Abr \n#> 9 si 0.938 0.0616 si 0.53 Sun Taxi no Mar Mar \n#> 10 si 0.931 0.0694 si 6.65 Taxicab In… no Dom Abr \n#> # ℹ 7,990 more rows\n#> # ℹ 1 more variable: hora <int>\n\nA esto le llamamos “resubstitución” ó “repredecir en los datos de entrenamiento”" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-1", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-1", + "title": "2 - Your data budget", + "section": "Data on tree frog hatching", + "text": "Data on tree frog hatching\n\nRed-eyed tree frog embryos can hatch earlier than their normal ~7 days if they detect potential predator threat!\nType ?stacks::tree_frogs to learn more about this dataset, including references.\nWe are using a slightly modified version from stacks.\n\n\nlibrary(tidymodels)\n\ndata(\"tree_frogs\", package = \"stacks\")\ntree_frogs <- tree_frogs %>%\n mutate(t_o_d = factor(t_o_d),\n age = age / 86400) %>%\n filter(!is.na(latency)) %>%\n select(-c(clutch, hatched))" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-4", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-4", - "title": "4 - Evaluar modelos", - "section": "Los peligros del sobreajustar ⚠️ ", - "text": "Los peligros del sobreajustar ⚠️ \n\ntaxi_ajustado %>%\n augment(taxi_entrenar) %>%\n accuracy(propina, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n\nA esto le llamamos “resubstitución de la estimación”" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-2", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-2", + "title": "2 - Your data budget", + "section": "Data on tree frog hatching", + "text": "Data on tree frog hatching\n\n\n\nN = 572\nA numeric outcome, latency\n4 other variables\n\ntreatment, reflex, and t_o_d are nominal predictors\nage is a numeric predictor\n\n\n\n\n\n\n\nlatency: How long it took the frog to hatch after being stimulated - i.e. after being poked by a blunt probe (in seconds).\ntreatment: Whether or not they got gentamicin, a compound that knocks out the embryo’s lateral line (a sensory organ).\nreflex: A measure of ear function (low, mid, full)\nt_o_d: Time that the stimulus was applied (morning, afternoon, night)\nage: Age at the time it was stimulated (in days)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-5", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-5", - "title": "4 - Evaluar modelos", - "section": "Los peligros del sobreajustar ⚠️ ", - "text": "Los peligros del sobreajustar ⚠️ \n\n\n\ntaxi_ajustado %>%\n augment(taxi_entrenar) %>%\n accuracy(propina, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-3", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-on-tree-frog-hatching-3", + "title": "2 - Your data budget", + "section": "Data on tree frog hatching", + "text": "Data on tree frog hatching\n\ntree_frogs\n#> # A tibble: 572 × 5\n#> treatment reflex age t_o_d latency\n#> <fct> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.40 morning 22\n#> 2 control low 4.18 night 360\n#> 3 control full 4.65 afternoon 106\n#> 4 control mid 4.14 night 180\n#> 5 control full 4.6 afternoon 60\n#> 6 gentamicin full 5.36 morning 39\n#> 7 control full 4.56 afternoon 214\n#> 8 control full 5.43 morning 50\n#> 9 control full 4.63 afternoon 224\n#> 10 control full 5.40 morning 63\n#> # ℹ 562 more rows" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-6", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-6", - "title": "4 - Evaluar modelos", - "section": "Los peligros del sobreajustar ⚠️ ", - "text": "Los peligros del sobreajustar ⚠️ \n\n\n\ntaxi_ajustado %>%\n augment(taxi_entrenar) %>%\n accuracy(propina, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n\n\n\ntaxi_ajustado %>%\n augment(taxi_prueba) %>%\n accuracy(propina, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.908\n\n\n\n⚠️ Acuerdate que estamos demonstrando el sobreajuste\n\n\n⚠️ No utilizes el set the prueba sino hasta el fin de tu analisis" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#ja-ja-ja-estoy-en-peligro", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#ja-ja-ja-estoy-en-peligro", - "title": "4 - Evaluar modelos", - "section": "“Ja-ja-ja, estoy en peligro”", - "text": "“Ja-ja-ja, estoy en peligro”" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-1", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-1", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-1", - "title": "4 - Evaluar modelos", - "section": "Tu turno", - "text": "Tu turno\n\nUsa augment() y una función de métrica para calcular una métrica de classificación, por ejemplo brier_class()\nCalcula las métricas para los datos de entrenamiento y de prueba para demonstrar el sobreajuste\nNota la evidencia de sobreajuste ⚠️\n\n\n\n−+\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-2", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-2", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-7", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#los-peligros-del-sobreajustar-7", - "title": "4 - Evaluar modelos", - "section": "Los peligros del sobreajustar ⚠️ ", - "text": "Los peligros del sobreajustar ⚠️ \n\n\n\ntaxi_ajustado %>%\n augment(taxi_entrenar) %>%\n brier_class(propina, .pred_si)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0632\n\n\n\ntaxi_ajustado %>%\n augment(taxi_prueba) %>%\n brier_class(propina, .pred_si)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0782\n\n\n\n¿Que tal si queremos comparar más modelos?\n\n\n…y comparar configuraciones para los modelos?\n\n\nY tambien queremos saber si las diferencias son importantes" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n03:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#remuestreo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#remuestreo", - "title": "4 - Evaluar modelos", - "section": "Remuestreo", - "text": "Remuestreo\n\n\n\n\n\nflowchart TD\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rm1[Remuestreo 1]\n style rm1 fill:#fff,stroke:#666,color:#000\n tr --> rm1\n rm2[Remuestreo 2]\n style rm2 fill:#fff,stroke:#666,color:#000\n tr --> rm2\n rm3[Remuestreo B]\n style rm3 fill:#fff,stroke:#666,color:#000\n tr --> rm3\n an1[Analysis]\n style an1 fill:#FBE9BF,stroke:#666,color:#000\n rm1 --> an1\n vl1[Validación]\n style vl1 fill:#E5E7FD,stroke:#666,color:#000\n rm1 --> vl1\n an2[Analysis]\n style an2 fill:#FBE9BF,stroke:#666,color:#000\n rm2 --> an2\n vl2[Validación]\n style vl2 fill:#E5E7FD,stroke:#666,color:#000\n rm2 --> vl2 \n an3[Analysis]\n style an3 fill:#FBE9BF,stroke:#666,color:#000\n rm3 --> an3\n vl3[Validación]\n style vl3 fill:#E5E7FD,stroke:#666,color:#000\n rm3 --> vl3" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-3", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-3", + "title": "2 - Your data budget", + "section": "Data splitting and spending ", + "text": "Data splitting and spending \n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs)\nfrog_split\n#> <Training/Testing/Total>\n#> <429/143/572>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada", - "title": "4 - Evaluar modelos", - "section": "Validación cruzada", - "text": "Validación cruzada" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#accessing-the-data", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#accessing-the-data", + "title": "2 - Your data budget", + "section": "Accessing the data ", + "text": "Accessing the data \n\nfrog_train <- training(frog_split)\nfrog_test <- testing(frog_split)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-1", - "title": "4 - Evaluar modelos", - "section": "Validación cruzada", - "text": "Validación cruzada" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#the-training-set", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#the-training-set", + "title": "2 - Your data budget", + "section": "The training set", + "text": "The training set\n\nfrog_train\n#> # A tibble: 429 × 5\n#> treatment reflex age t_o_d latency\n#> <fct> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.36 morning 36\n#> 2 gentamicin full 5.37 morning 72\n#> 3 gentamicin full 4.65 afternoon 141\n#> 4 control full 5.42 morning 27\n#> 5 control full 5.43 morning 27\n#> 6 gentamicin full 5.38 morning 73\n#> 7 gentamicin full 5.42 morning 68\n#> 8 gentamicin full 4.75 afternoon 124\n#> 9 control full 5.00 night 62\n#> 10 control full 5.39 morning 25\n#> # ℹ 419 more rows" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-2", - "title": "4 - Evaluar modelos", - "section": "Tu turno", - "text": "Tu turno\n\nSi usamos 10 plieges (folds), cual es el porcentaje de datos de entrenamiento\n\ncuantos terminan en análisis\ncuantos terminan en evaluación (assesment)\n\n…para cada pliege?\n\n\n\n\n−+\n03:00" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#the-test-set", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#the-test-set", + "title": "2 - Your data budget", + "section": "The test set ", + "text": "The test set \n\nfrog_test\n#> # A tibble: 143 × 5\n#> treatment reflex age t_o_d latency\n#> <fct> <fct> <dbl> <fct> <dbl>\n#> 1 control full 5.40 morning 22\n#> 2 control low 4.18 night 360\n#> 3 control full 4.63 afternoon 224\n#> 4 gentamicin full 4.75 afternoon 158\n#> 5 control mid 4.22 night 91\n#> 6 gentamicin full 4.89 night 301\n#> 7 control full 5.38 morning 2\n#> 8 control full 4.80 afternoon 56\n#> 9 control full 5.36 morning 11\n#> 10 control full 5.40 morning 64\n#> # ℹ 133 more rows" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-2", - "title": "4 - Evaluar modelos", - "section": "Validación cruzada ", - "text": "Validación cruzada \n\nvfold_cv(taxi_entrenar) \n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn-1", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn-1", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n05:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-3", - "title": "4 - Evaluar modelos", - "section": "Validación cruzada ", - "text": "Validación cruzada \n¿Que hay en este?\n\ntaxi_plieges <- vfold_cv(taxi_entrenar)\ntaxi_plieges$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n\n\nTalk about a list column, storing non-atomic types in dataframe" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-4", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#data-splitting-and-spending-4", + "title": "2 - Your data budget", + "section": "Data splitting and spending ", + "text": "Data splitting and spending \n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs, prop = 0.8)\nfrog_train <- training(frog_split)\nfrog_test <- testing(frog_split)\n\nnrow(frog_train)\n#> [1] 457\nnrow(frog_test)\n#> [1] 115" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-4", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-4", - "title": "4 - Evaluar modelos", - "section": "Validación cruzada ", - "text": "Validación cruzada \n\nvfold_cv(taxi_entrenar, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [6400/1600]> Fold1\n#> 2 <split [6400/1600]> Fold2\n#> 3 <split [6400/1600]> Fold3\n#> 4 <split [6400/1600]> Fold4\n#> 5 <split [6400/1600]> Fold5" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-1", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-1", + "title": "2 - Your data budget", + "section": "", + "text": "We will use this tomorrow" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-5", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-5", - "title": "4 - Evaluar modelos", - "section": "Validación cruzada ", - "text": "Validación cruzada \n\nvfold_cv(taxi_entrenar, strata = propina)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10\n\n\nEstratificar usualmente ayuda, y con pocos malas consecuencias" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn-2", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#your-turn-2", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nExplore the frog_train data on your own!\n\nWhat’s the distribution of the outcome, latency?\nWhat’s the distribution of numeric variables like age?\nHow does latency differ across the categorical variables?\n\n\n\n\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-6", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-6", - "title": "4 - Evaluar modelos", - "section": "Validación cruzada ", - "text": "Validación cruzada \nUsaremos esto:\n\nset.seed(123)\ntaxi_plieges <- vfold_cv(taxi_entrenar, v = 10, strata = propina)\ntaxi_plieges\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10\n\n\nEspecifica la semilla (seed) cuando estas creando remuestreos" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-3", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-3", + "title": "2 - Your data budget", + "section": "", + "text": "ggplot(frog_train, aes(latency)) +\n geom_histogram(bins = 20)\n\n\n\nThis histogram brings up a concern. What if in our training set we get unlucky and sample few or none of these large values? That could mean that our model wouldn’t be able to predict such values. Let’s come back to that!" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#ajustemos-nuestro-modelo-usando-los-remuestreos", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#ajustemos-nuestro-modelo-usando-los-remuestreos", - "title": "4 - Evaluar modelos", - "section": "Ajustemos nuestro modelo usando los remuestreos", - "text": "Ajustemos nuestro modelo usando los remuestreos\n\ntaxi_res <- fit_resamples(arbol_flujo, taxi_plieges)\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]>" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-4", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-4", + "title": "2 - Your data budget", + "section": "", + "text": "ggplot(frog_train, aes(latency, treatment, fill = treatment)) +\n geom_boxplot(alpha = 0.5, show.legend = FALSE)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo", - "title": "4 - Evaluar modelos", - "section": "Evaluando la calidad del modelo ", - "text": "Evaluando la calidad del modelo \n\ntaxi_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.915 10 0.00309 Preprocessor1_Model1\n#> 2 roc_auc binary 0.624 10 0.0105 Preprocessor1_Model1\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nPodemos medir correctamente la la calidad del modelo usando solo los datos de entrenamiento 🎉" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-5", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-5", + "title": "2 - Your data budget", + "section": "", + "text": "frog_train %>%\n ggplot(aes(latency, reflex, fill = reflex)) +\n geom_boxplot(alpha = 0.3, show.legend = FALSE)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#comparando-las-métricas", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#comparando-las-métricas", - "title": "4 - Evaluar modelos", - "section": "Comparando las métricas ", - "text": "Comparando las métricas \n¿Que diferencia hay entre las métricas usando los datos de remuestreo, y usando los datos de entrenamiento y prueba?\n\n\n\ntaxi_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.915 10\n#> 2 roc_auc 0.624 10\n\n\nEl ROC AUC antes era:\n\n0.69 para el set the entrenamiento\n0.64 para el set the prueba\n\n\n\nAcuerdate que:\n⚠️ Los datos de entrenamiento da métricas demasiado optimísticas\n⚠️ Los datos de prueba son valiosos" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-6", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-6", + "title": "2 - Your data budget", + "section": "", + "text": "ggplot(frog_train, aes(age, latency, color = reflex)) +\n geom_point(alpha = .8, size = 2)" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-1", - "title": "4 - Evaluar modelos", - "section": "Evaluando la calidad del modelo ", - "text": "Evaluando la calidad del modelo \n\n\nctrl_taxi <- control_resamples(save_pred = TRUE)\ntaxi_res <- fit_resamples(arbol_flujo, taxi_plieges, control = ctrl_taxi)\n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-7", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#section-7", + "title": "2 - Your data budget", + "section": "", + "text": "Stratified sampling would split within each quartile\n\nBased on our exploration, we realized that stratifying by latency might help get a consistent distribution. For instance, we’d include high and low latency in both the test and training" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-2", - "title": "4 - Evaluar modelos", - "section": "Evaluando la calidad del modelo ", - "text": "Evaluando la calidad del modelo \n\n# Guarde los resultados de las evaluaciones\ntaxi_preds <- collect_predictions(taxi_res)\ntaxi_preds\n#> # A tibble: 8,000 × 7\n#> id .pred_si .pred_no .row .pred_class propina .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 Fold01 0.938 0.0615 14 si si Preprocessor1_Model1\n#> 2 Fold01 0.946 0.0544 19 si si Preprocessor1_Model1\n#> 3 Fold01 0.973 0.0269 33 si si Preprocessor1_Model1\n#> 4 Fold01 0.903 0.0971 43 si si Preprocessor1_Model1\n#> 5 Fold01 0.973 0.0269 74 si si Preprocessor1_Model1\n#> 6 Fold01 0.903 0.0971 103 si si Preprocessor1_Model1\n#> 7 Fold01 0.915 0.0851 104 si no Preprocessor1_Model1\n#> 8 Fold01 0.903 0.0971 124 si si Preprocessor1_Model1\n#> 9 Fold01 0.667 0.333 126 si si Preprocessor1_Model1\n#> 10 Fold01 0.949 0.0510 128 si si Preprocessor1_Model1\n#> # ℹ 7,990 more rows" + "objectID": "archive/2022-08-Reykjavik-City/02-data-budget.html#stratification", + "href": "archive/2022-08-Reykjavik-City/02-data-budget.html#stratification", + "title": "2 - Your data budget", + "section": "Stratification", + "text": "Stratification\nUse strata = latency\n\nset.seed(123)\nfrog_split <- initial_split(tree_frogs, prop = 0.8, strata = latency)\nfrog_split\n#> <Training/Testing/Total>\n#> <456/116/572>\n\n\nStratification often helps, with very little downside" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-3", - "title": "4 - Evaluar modelos", - "section": "Evaluando la calidad del modelo ", - "text": "Evaluando la calidad del modelo \n\ntaxi_preds %>% \n group_by(id) %>%\n taxi_metrics(truth = propina, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.905\n#> 2 Fold02 accuracy binary 0.925\n#> 3 Fold03 accuracy binary 0.926\n#> 4 Fold04 accuracy binary 0.915\n#> 5 Fold05 accuracy binary 0.902\n#> 6 Fold06 accuracy binary 0.912\n#> 7 Fold07 accuracy binary 0.906\n#> 8 Fold08 accuracy binary 0.91 \n#> 9 Fold09 accuracy binary 0.918\n#> 10 Fold10 accuracy binary 0.931\n#> # ℹ 20 more rows" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 rsq standard 0.380\n#> 3 mae standard 40.2\n\n\n\nRMSE: difference between the predicted and observed values ⬇️\n\\(R^2\\): squared correlation between the predicted and observed values ⬆️\nMAE: similar to RMSE, but mean absolute error ⬇️" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#donde-estan-los-modelos-que-ajustamos", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#donde-estan-los-modelos-que-ajustamos", - "title": "4 - Evaluar modelos", - "section": "¿Donde estan los modelos que ajustamos? ", - "text": "¿Donde estan los modelos que ajustamos? \n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-1", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bootstrapping", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bootstrapping", - "title": "4 - Evaluar modelos", - "section": "Bootstrapping", - "text": "Bootstrapping" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-2", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-2", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\naugment(tree_fit, new_data = frog_test) %>%\n group_by(reflex) %>%\n rmse(latency, .pred)\n#> # A tibble: 3 × 4\n#> reflex .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 low rmse standard 94.3\n#> 2 mid rmse standard 101. \n#> 3 full rmse standard 51.2" }, - { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bootstrapping-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bootstrapping-1", - "title": "4 - Evaluar modelos", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(taxi_entrenar)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [8000/2902]> Bootstrap01\n#> 2 <split [8000/2916]> Bootstrap02\n#> 3 <split [8000/3004]> Bootstrap03\n#> 4 <split [8000/2979]> Bootstrap04\n#> 5 <split [8000/2961]> Bootstrap05\n#> 6 <split [8000/2962]> Bootstrap06\n#> 7 <split [8000/3026]> Bootstrap07\n#> 8 <split [8000/2926]> Bootstrap08\n#> 9 <split [8000/2972]> Bootstrap09\n#> 10 <split [8000/2972]> Bootstrap10\n#> # ℹ 15 more rows" + { + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-3", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#metrics-for-model-performance-3", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\nfrog_metrics <- metric_set(rmse, msd)\naugment(tree_fit, new_data = frog_test) %>%\n frog_metrics(latency, .pred)\n#> # A tibble: 2 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 msd standard -0.908" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---donde-estamos", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---donde-estamos", - "title": "4 - Evaluar modelos", - "section": "Expectativas del taller - Donde estamos", - "text": "Expectativas del taller - Donde estamos\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestras]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#fff,stroke:#eee,color:#ddd\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#fff,stroke:#eee,color:#ddd\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#fff,stroke:#eee,color:#ddd\n fm --> vm\n ts --> vm" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-1", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-3", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-3", - "title": "4 - Evaluar modelos", - "section": "Tu turno", - "text": "Tu turno\n\nCrea un:\n\nSet de validación cruzada tipo Monte Carlo\nSet de validación\n\nNo te olvides de usar set.seed()\n\n\n\n−+\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-2", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-2", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-tipo-monte-carlo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#validación-cruzada-tipo-monte-carlo", - "title": "4 - Evaluar modelos", - "section": "Validación cruzada tipo Monte Carlo ", - "text": "Validación cruzada tipo Monte Carlo \n\nset.seed(322)\nmc_cv(taxi_entrenar, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6000/2000]> Resample01\n#> 2 <split [6000/2000]> Resample02\n#> 3 <split [6000/2000]> Resample03\n#> 4 <split [6000/2000]> Resample04\n#> 5 <split [6000/2000]> Resample05\n#> 6 <split [6000/2000]> Resample06\n#> 7 <split [6000/2000]> Resample07\n#> 8 <split [6000/2000]> Resample08\n#> 9 <split [6000/2000]> Resample09\n#> 10 <split [6000/2000]> Resample10" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-3", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-3", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\ntree_fit %>%\n augment(frog_train)\n#> # A tibble: 456 × 6\n#> treatment reflex age t_o_d latency .pred\n#> <fct> <fct> <dbl> <fct> <dbl> <dbl>\n#> 1 control full 5.42 morning 33 39.8\n#> 2 control full 5.38 morning 19 66.7\n#> 3 control full 5.38 morning 2 66.7\n#> 4 control full 5.44 morning 39 39.8\n#> 5 control full 5.41 morning 42 39.8\n#> 6 control full 4.75 afternoon 20 59.8\n#> 7 control full 4.95 night 31 83.1\n#> 8 control full 5.42 morning 21 39.8\n#> 9 gentamicin full 5.39 morning 30 64.6\n#> 10 control full 4.55 afternoon 43 174. \n#> # ℹ 446 more rows\n\nWe call this “resubstitution” or “repredicting the training set”" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#set-de-validación", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#set-de-validación", - "title": "4 - Evaluar modelos", - "section": "Set de validación ", - "text": "Set de validación \n\nset.seed(853)\ntaxi_val_split <- initial_validation_split(taxi, strata = propina)\nvalidation_set(taxi_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6000/2000]> validation\n\n\nUn set de validación es solamente otro tipo de remuestreo" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-4", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-4", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4\n\nWe call this a “resubstitution estimate”" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bosque-aleatorio-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#bosque-aleatorio-1", - "title": "4 - Evaluar modelos", - "section": "Bosque aleatorio 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Bosque aleatorio 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsambla varios árboles de decisión\n¡Todos los árboles votan! 🗳️\nAgregación tipo bootstrap + muestreo aleatorio del predictor\n\n\n\nUsualmente funciona bien sin usar afinamiento, pero solo mientras hayan suficientes árboles" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-5", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-5", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#crear-un-modelo-de-bosque-aleatorio", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#crear-un-modelo-de-bosque-aleatorio", - "title": "4 - Evaluar modelos", - "section": "Crear un modelo de bosque aleatorio ", - "text": "Crear un modelo de bosque aleatorio \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-6", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-6", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4\n\n\n\ntree_fit %>%\n augment(frog_test) %>%\n rmse(latency, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#crear-un-modelo-de-bosque-aleatorio-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#crear-un-modelo-de-bosque-aleatorio-1", - "title": "4 - Evaluar modelos", - "section": "Crear un modelo de bosque aleatorio ", - "text": "Crear un modelo de bosque aleatorio \n\nrf_wflow <- workflow(propina ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse augment() and metrics() to compute a regression metric like mae().\nCompute the metrics for both training and testing data.\nNotice the evidence of overfitting! ⚠️\n\n\n\n05:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-4", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#tu-turno-4", - "title": "4 - Evaluar modelos", - "section": "Tu turno", - "text": "Tu turno\n\nUsa fit_resamples() y rf_wflow para lo siguiente:\n\nQuedarse con las predicciones\nCalcular las métricas\n\n\n\n\n−+\n08:00" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-7", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#dangers-of-overfitting-7", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntree_fit %>%\n augment(frog_train) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 49.4 \n#> 2 rsq standard 0.494\n#> 3 mae standard 33.4\n\n\n\ntree_fit %>%\n augment(frog_test) %>%\n metrics(latency, .pred)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 rmse standard 59.2 \n#> 2 rsq standard 0.380\n#> 3 mae standard 40.2\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-4", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#evaluando-la-calidad-del-modelo-4", - "title": "4 - Evaluar modelos", - "section": "Evaluando la calidad del modelo ", - "text": "Evaluando la calidad del modelo \n\nctrl_taxi <- control_resamples(save_pred = TRUE)\n\n# Bosque aleatorio usan numeros al azar, asi que asegurate de definir la semilla\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, taxi_plieges, control = ctrl_taxi)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.924 10 0.00323 Preprocessor1_Model1\n#> 2 roc_auc binary 0.611 10 0.0149 Preprocessor1_Model1" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---donde-estamos-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---donde-estamos-1", - "title": "4 - Evaluar modelos", - "section": "Expectativas del taller - Donde estamos", - "text": "Expectativas del taller - Donde estamos\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestras]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#fff,stroke:#eee,color:#ddd\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#fff,stroke:#eee,color:#ddd\n fm --> vm\n ts --> vm" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-1", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#el-ajuste-final", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#el-ajuste-final", - "title": "4 - Evaluar modelos", - "section": "El ajuste final ", - "text": "El ajuste final \nDigamos que estamos satisfechos con usar nuestro modelo de bosque aleatorio\nAjustemos el modelo usando todos los datos en el set the entrenamiento, y despues midamos la calidad del modelo con el set the prueba\n\nHemos usado fit() y predict() (+ augment()), pero hay un atajo:\n\n# taxi_separar has train + test info\najuste_final <- last_fit(rf_wflow, taxi_separar) \n\najuste_final\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [8000/2000]> train/test split <tibble> <tibble> <tibble> <workflow>" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-1", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n03:00" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final", - "title": "4 - Evaluar modelos", - "section": "¿Que contiene ajuste_final? ", - "text": "¿Que contiene ajuste_final? \n\ncollect_metrics(ajuste_final)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.914 Preprocessor1_Model1\n#> 2 roc_auc binary 0.630 Preprocessor1_Model1\n\n\nLas métricas fueron calculadas con el set de prueba" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-2", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-2", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(frog_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [410/46]> Fold01\n#> 2 <split [410/46]> Fold02\n#> 3 <split [410/46]> Fold03\n#> 4 <split [410/46]> Fold04\n#> 5 <split [410/46]> Fold05\n#> 6 <split [410/46]> Fold06\n#> 7 <split [411/45]> Fold07\n#> 8 <split [411/45]> Fold08\n#> 9 <split [411/45]> Fold09\n#> 10 <split [411/45]> Fold10" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final-1", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final-1", - "title": "4 - Evaluar modelos", - "section": "¿Que contiene ajuste_final? ", - "text": "¿Que contiene ajuste_final? \n\ncollect_predictions(ajuste_final)\n#> # A tibble: 2,000 × 7\n#> id .pred_si .pred_no .row .pred_class propina .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 train/test split 0.949 0.0513 4 si si Preprocessor1_M…\n#> 2 train/test split 0.922 0.0777 10 si si Preprocessor1_M…\n#> 3 train/test split 0.961 0.0395 19 si si Preprocessor1_M…\n#> 4 train/test split 0.889 0.111 23 si si Preprocessor1_M…\n#> 5 train/test split 0.936 0.0639 28 si si Preprocessor1_M…\n#> 6 train/test split 0.969 0.0306 34 si si Preprocessor1_M…\n#> 7 train/test split 0.962 0.0377 35 si si Preprocessor1_M…\n#> 8 train/test split 0.939 0.0606 38 si si Preprocessor1_M…\n#> 9 train/test split 0.986 0.0138 40 si si Preprocessor1_M…\n#> 10 train/test split 0.957 0.0431 42 si no Preprocessor1_M…\n#> # ℹ 1,990 more rows" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-3", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-3", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWhat is in this?\n\nfrog_folds <- vfold_cv(frog_train)\nfrog_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <410/46/456>\n\n\nTalk about a list column, storing non-atomic types in dataframe" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final-2", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#que-contiene-ajuste_final-2", - "title": "4 - Evaluar modelos", - "section": "¿Que contiene ajuste_final? ", - "text": "¿Que contiene ajuste_final? \n\nextract_workflow(ajuste_final)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 8000 \n#> Number of independent variables: 6 \n#> Mtry: 2 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.07070342\n\n\nUse this for prediction on new data, like for deploying" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-4", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-4", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(frog_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [364/92]> Fold1\n#> 2 <split [365/91]> Fold2\n#> 3 <split [365/91]> Fold3\n#> 4 <split [365/91]> Fold4\n#> 5 <split [365/91]> Fold5" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---ya-llegamos", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-04-evaluar-modelos.html#expectativas-del-taller---ya-llegamos", - "title": "4 - Evaluar modelos", - "section": "Expectativas del taller - ¡Ya llegamos!", - "text": "Expectativas del taller - ¡Ya llegamos!\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#FBE9BF,stroke:#666,color:#000\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#E5E7FD,stroke:#666,color:#000\n fm --> vm\n ts --> vm" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-5", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-5", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(frog_train, strata = latency)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [408/48]> Fold01\n#> 2 <split [408/48]> Fold02\n#> 3 <split [408/48]> Fold03\n#> 4 <split [409/47]> Fold04\n#> 5 <split [411/45]> Fold05\n#> 6 <split [412/44]> Fold06\n#> 7 <split [412/44]> Fold07\n#> 8 <split [412/44]> Fold08\n#> 9 <split [412/44]> Fold09\n#> 10 <split [412/44]> Fold10\n\n\nStratification often helps, with very little downside" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-06-para-terminar.html#tu-turno", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-06-para-terminar.html#tu-turno", - "title": "6 - Para terminar", - "section": "Tu turno", - "text": "Tu turno\n\n¿Que fue algo que te sorprendio aprender?\n¿Cual será lo que aprendiste que planeas usar?\n\n\n\n−+\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-6", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#cross-validation-6", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nfrog_folds <- vfold_cv(frog_train, v = 10, strata = latency)\nfrog_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [408/48]> Fold01\n#> 2 <split [408/48]> Fold02\n#> 3 <split [408/48]> Fold03\n#> 4 <split [409/47]> Fold04\n#> 5 <split [411/45]> Fold05\n#> 6 <split [412/44]> Fold06\n#> 7 <split [412/44]> Fold07\n#> 8 <split [412/44]> Fold08\n#> 9 <split [412/44]> Fold09\n#> 10 <split [412/44]> Fold10\n\n\nSet the seed when creating resamples" }, { - "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-06-para-terminar.html#recursos-para-seguir-aprendiendo", - "href": "archive/2024-03-conectaR-spanish/presentacion/intro-06-para-terminar.html#recursos-para-seguir-aprendiendo", - "title": "6 - Para terminar", - "section": "Recursos para seguir aprendiendo", - "text": "Recursos para seguir aprendiendo\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#fit-our-model-to-the-resamples", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#fit-our-model-to-the-resamples", + "title": "4 - Evaluating models", + "section": "Fit our model to the resamples", + "text": "Fit our model to the resamples\n\ntree_res <- fit_resamples(tree_wflow, frog_folds)\ntree_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]>" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#working-with-our-predictors", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#working-with-our-predictors", - "title": "2 - Feature Engineering", - "section": "Working with our predictors", - "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for linear regression).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\ntree_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 rmse standard 59.6 10 2.31 Preprocessor1_Model1\n#> 2 rsq standard 0.305 10 0.0342 Preprocessor1_Model1\n\n\nWe can reliably measure performance using only the training data 🎉" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#what-is-feature-engineering", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#what-is-feature-engineering", - "title": "2 - Feature Engineering", - "section": "What is feature engineering?", - "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPrincipal component analysis (PCA) feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection (FES) and Applied Machine Learning for Tabular Data (aml4td)." + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#comparing-metrics", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#comparing-metrics", + "title": "4 - Evaluating models", + "section": "Comparing metrics ", + "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\ntree_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 rmse 59.6 10\n#> 2 rsq 0.305 10\n\n\nThe RMSE previously was\n\n49.36 for the training set\n59.16 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#example-dates", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#example-dates", - "title": "2 - Feature Engineering", - "section": "Example: Dates", - "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance-1", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_frog <- control_resamples(save_pred = TRUE)\ntree_res <- fit_resamples(tree_wflow, frog_folds, control = ctrl_frog)\n\ntree_preds <- collect_predictions(tree_res)\ntree_preds\n#> # A tibble: 456 × 5\n#> id .pred .row latency .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 Fold01 39.6 1 33 Preprocessor1_Model1\n#> 2 Fold01 72.1 3 2 Preprocessor1_Model1\n#> 3 Fold01 63.8 9 30 Preprocessor1_Model1\n#> 4 Fold01 72.1 13 46 Preprocessor1_Model1\n#> 5 Fold01 43.3 28 11 Preprocessor1_Model1\n#> 6 Fold01 61.7 35 41 Preprocessor1_Model1\n#> 7 Fold01 39.6 51 43 Preprocessor1_Model1\n#> 8 Fold01 134. 70 20 Preprocessor1_Model1\n#> 9 Fold01 70.6 74 21 Preprocessor1_Model1\n#> 10 Fold01 39.6 106 14 Preprocessor1_Model1\n#> # ℹ 446 more rows" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#general-definitions", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#general-definitions", - "title": "2 - Feature Engineering", - "section": "General definitions ", - "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-3", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-3", + "title": "4 - Evaluating models", + "section": "", + "text": "tree_preds %>% \n ggplot(aes(latency, .pred, color = id)) + \n geom_abline(lty = 2, col = \"gray\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#previously---setup", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#previously---setup", - "title": "2 - Feature Engineering", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\n\n# Add another package:\nlibrary(textrecipes)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#where-are-the-fitted-models", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#where-are-the-fitted-models", + "title": "4 - Evaluating models", + "section": "Where are the fitted models? ", + "text": "Where are the fitted models? \n\ntree_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [408/48]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 2 <split [408/48]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 3 <split [408/48]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [48 × 4]>\n#> 4 <split [409/47]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [47 × 4]>\n#> 5 <split [411/45]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [45 × 4]>\n#> 6 <split [412/44]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 7 <split [412/44]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 8 <split [412/44]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 9 <split [412/44]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n#> 10 <split [412/44]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble [44 × 4]>\n\n\n🗑️" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#previously---data-usage", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#previously---data-usage", - "title": "2 - Feature Engineering", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\n\nWe’ll go from here and create a set of resamples to use for model assessments." + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping", + "title": "4 - Evaluating models", + "section": "Bootstrapping", + "text": "Bootstrapping" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#resampling-strategy", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#resampling-strategy", - "title": "2 - Feature Engineering", - "section": "Resampling Strategy", - "text": "Resampling Strategy" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping-1", + "title": "4 - Evaluating models", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(frog_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [456/163]> Bootstrap01\n#> 2 <split [456/166]> Bootstrap02\n#> 3 <split [456/173]> Bootstrap03\n#> 4 <split [456/177]> Bootstrap04\n#> 5 <split [456/166]> Bootstrap05\n#> 6 <split [456/163]> Bootstrap06\n#> 7 <split [456/164]> Bootstrap07\n#> 8 <split [456/165]> Bootstrap08\n#> 9 <split [456/170]> Bootstrap09\n#> 10 <split [456/177]> Bootstrap10\n#> # ℹ 15 more rows" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#resampling-strategy-1", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#resampling-strategy-1", - "title": "2 - Feature Engineering", - "section": "Resampling Strategy ", - "text": "Resampling Strategy \nWe’ll use simple 10-fold cross-validation (stratified sampling):\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)\nhotel_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [3372/377]> Fold01\n#> 2 <split [3373/376]> Fold02\n#> 3 <split [3373/376]> Fold03\n#> 4 <split [3373/376]> Fold04\n#> 5 <split [3373/376]> Fold05\n#> 6 <split [3374/375]> Fold06\n#> 7 <split [3375/374]> Fold07\n#> 8 <split [3376/373]> Fold08\n#> 9 <split [3376/373]> Fold09\n#> 10 <split [3376/373]> Fold10" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-2", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-2", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCreate:\n\nbootstrap folds (change times from the default)\nvalidation set (use the reference guide to find the function)\n\nDon’t forget to set a seed when you resample!\n\n\n\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", - "title": "2 - Feature Engineering", - "section": "Prepare your data for modeling ", - "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of preprocessing and feature engineering steps.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping-2", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#bootstrapping-2", + "title": "4 - Evaluating models", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(322)\nbootstraps(frog_train, times = 10)\n#> # Bootstrap sampling \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [456/173]> Bootstrap01\n#> 2 <split [456/168]> Bootstrap02\n#> 3 <split [456/170]> Bootstrap03\n#> 4 <split [456/164]> Bootstrap04\n#> 5 <split [456/176]> Bootstrap05\n#> 6 <split [456/156]> Bootstrap06\n#> 7 <split [456/166]> Bootstrap07\n#> 8 <split [456/168]> Bootstrap08\n#> 9 <split [456/167]> Bootstrap09\n#> 10 <split [456/170]> Bootstrap10" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#a-first-recipe", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#a-first-recipe", - "title": "2 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#validation-set", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#validation-set", + "title": "4 - Evaluating models", + "section": "Validation set ", + "text": "Validation set \n\nset.seed(853)\nvalidation_split(frog_train, strata = latency)\n#> # Validation Set Split (0.75/0.25) using stratification \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [340/116]> validation\n\n\nA validation set is just another type of resample" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#a-first-recipe-1", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#a-first-recipe-1", - "title": "2 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nsummary(hotel_rec)\n#> # A tibble: 27 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 lead_time <chr [2]> predictor original\n#> 2 stays_in_weekend_nights <chr [2]> predictor original\n#> 3 stays_in_week_nights <chr [2]> predictor original\n#> 4 adults <chr [2]> predictor original\n#> 5 children <chr [2]> predictor original\n#> 6 babies <chr [2]> predictor original\n#> 7 meal <chr [3]> predictor original\n#> 8 country <chr [3]> predictor original\n#> 9 market_segment <chr [3]> predictor original\n#> 10 distribution_channel <chr [3]> predictor original\n#> # ℹ 17 more rows\n\nThe type column contains information on the variables" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#random-forest-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#random-forest-1", + "title": "4 - Evaluating models", + "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this tomorrow!), as long as there are enough trees" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn", - "title": "2 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\nWhat do you think are in the type vectors for the lead_time and country columns?\n\n\n\n−+\n02:00" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#create-a-random-forest-model", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#create-a-random-forest-model", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"regression\")\nrf_spec\n#> Random Forest Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#create-indicator-variables", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#create-indicator-variables", - "title": "2 - Feature Engineering", - "section": "Create indicator variables ", - "text": "Create indicator variables \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns.\nstep_dummy() records the levels of the categorical predictors in the training set." + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#create-a-random-forest-model-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#create-a-random-forest-model-1", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_wflow <- workflow(latency ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#filter-out-constant-columns", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#filter-out-constant-columns", - "title": "2 - Feature Engineering", - "section": "Filter out constant columns ", - "text": "Filter out constant columns \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-3", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-3", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\nplot true vs. predicted values\n\n\n\n\n08:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#normalization", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#normalization", - "title": "2 - Feature Engineering", - "section": "Normalization ", - "text": "Normalization \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance-2", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluating-model-performance-2", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nctrl_frog <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, frog_folds, control = ctrl_frog)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 rmse standard 55.9 10 1.76 Preprocessor1_Model1\n#> 2 rsq standard 0.372 10 0.0312 Preprocessor1_Model1" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#reduce-correlation", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#reduce-correlation", - "title": "2 - Feature Engineering", - "section": "Reduce correlation ", - "text": "Reduce correlation \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-5", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-5", + "title": "4 - Evaluating models", + "section": "", + "text": "collect_predictions(rf_res) %>% \n ggplot(aes(latency, .pred, color = id)) + \n geom_abline(lty = 2, col = \"gray\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps", - "title": "2 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", + "title": "4 - Evaluating models", + "section": "How can we compare multiple model workflows at once?", + "text": "How can we compare multiple model workflows at once?" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-1", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-1", - "title": "2 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = vars(avg_price_per_room))\n\n\nA fancy machine learning supervised dimension reduction technique called UMAP…\n\nNote that this uses the outcome, and it is from an extension package" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set", + "title": "4 - Evaluating models", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec))\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-2", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#other-possible-steps-2", - "title": "2 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set-1", + "title": "4 - Evaluating models", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = frog_folds)\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn-1", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn-1", - "title": "2 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\n\nCreate a recipe() for the hotel data to:\n\nuse a Yeo-Johnson (YJ) transformation on lead_time\nconvert factors to indicator variables\nremove zero-variance variables\nadd the spline technique shown above\n\n\n\n\n−+\n03:00" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set-2", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#evaluate-a-workflow-set-2", + "title": "4 - Evaluating models", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nworkflow_set(list(latency ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = frog_folds) %>%\n rank_results()\n#> # A tibble: 4 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_fo… Prepro… rmse 55.7 1.81 10 formula rand… 1\n#> 2 formula_rand_fo… Prepro… rsq 0.375 0.0309 10 formula rand… 1\n#> 3 formula_decisio… Prepro… rmse 59.6 2.31 10 formula deci… 2\n#> 4 formula_decisio… Prepro… rsq 0.305 0.0342 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#minimal-recipe", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#minimal-recipe", - "title": "2 - Feature Engineering", - "section": "Minimal recipe ", - "text": "Minimal recipe \n\nhotel_indicators <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-4", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-4", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nWhen do you think a workflow set would be useful?\n\n\n\n03:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#measuring-performance", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#measuring-performance", - "title": "2 - Feature Engineering", - "section": "Measuring Performance ", - "text": "Measuring Performance \nWe’ll compute two measures: mean absolute error and the coefficient of determination (a.k.a \\(R^2\\)).\n\\[\\begin{align}\nMAE &= \\frac{1}{n}\\sum_{i=1}^n |y_i - \\hat{y}_i| \\notag \\\\\nR^2 &= cor(y_i, \\hat{y}_i)^2\n\\end{align}\\]\nThe focus will be on MAE for parameter optimization. We’ll use a metric set to compute these:\n\nreg_metrics <- metric_set(mae, rsq)" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#the-final-fit", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#the-final-fit", + "title": "4 - Evaluating models", + "section": "The final fit ", + "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# frog_split has train + test info\nfinal_fit <- last_fit(rf_wflow, frog_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [456/116]> train/test split <tibble> <tibble> <tibble> <workflow>" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#using-a-workflow", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#using-a-workflow", - "title": "2 - Feature Engineering", - "section": "Using a workflow ", - "text": "Using a workflow \n\nset.seed(9)\n\nhotel_lm_wflow <-\n workflow() %>%\n add_recipe(hotel_indicators) %>%\n add_model(linear_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nhotel_lm_res <-\n hotel_lm_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_lm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.6 10 0.214 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00339 Preprocessor1_Model1" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 rmse standard 57.3 Preprocessor1_Model1\n#> 2 rsq standard 0.415 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn-2", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#your-turn-2", - "title": "2 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n−+\n05:00" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit-1", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit-1", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 116 × 5\n#> id .pred .row latency .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 train/test split 44.1 1 22 Preprocessor1_Model1\n#> 2 train/test split 101. 3 106 Preprocessor1_Model1\n#> 3 train/test split 75.7 6 39 Preprocessor1_Model1\n#> 4 train/test split 43.1 8 50 Preprocessor1_Model1\n#> 5 train/test split 43.5 10 63 Preprocessor1_Model1\n#> 6 train/test split 43.8 14 25 Preprocessor1_Model1\n#> 7 train/test split 51.2 16 48 Preprocessor1_Model1\n#> 8 train/test split 161. 17 91 Preprocessor1_Model1\n#> 9 train/test split 50.9 32 11 Preprocessor1_Model1\n#> 10 train/test split 177. 33 109 Preprocessor1_Model1\n#> # ℹ 106 more rows\n\n\nThese are predictions for the test set" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#holdout-predictions", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#holdout-predictions", - "title": "2 - Feature Engineering", - "section": "Holdout predictions ", - "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nlm_cv_pred <- collect_predictions(hotel_lm_res)\nlm_cv_pred %>% print(n = 7)\n#> # A tibble: 3,749 × 5\n#> .pred id .row avg_price_per_room .config \n#> <dbl> <chr> <int> <dbl> <chr> \n#> 1 75.1 Fold01 20 40 Preprocessor1_Model1\n#> 2 49.3 Fold01 28 54 Preprocessor1_Model1\n#> 3 64.9 Fold01 45 50 Preprocessor1_Model1\n#> 4 52.8 Fold01 49 42 Preprocessor1_Model1\n#> 5 48.6 Fold01 61 49 Preprocessor1_Model1\n#> 6 29.8 Fold01 66 40 Preprocessor1_Model1\n#> 7 36.9 Fold01 88 49 Preprocessor1_Model1\n#> # ℹ 3,742 more rows" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-6", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#section-6", + "title": "4 - Evaluating models", + "section": "", + "text": "collect_predictions(final_fit) %>%\n ggplot(aes(latency, .pred)) + \n geom_abline(lty = 2, col = \"deeppink4\", size = 1.5) +\n geom_point(alpha = 0.5) +\n coord_obs_pred()" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#calibration-plot", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#calibration-plot", - "title": "2 - Feature Engineering", - "section": "Calibration Plot ", - "text": "Calibration Plot \n\nlibrary(probably)\n\ncal_plot_regression(hotel_lm_res)" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit-2", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#what-is-in-final_fit-2", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> latency ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1)) \n#> \n#> Type: Regression \n#> Number of trees: 1000 \n#> Sample size: 456 \n#> Number of independent variables: 4 \n#> Mtry: 2 \n#> Target node size: 5 \n#> Variable importance mode: none \n#> Splitrule: variance \n#> OOB prediction error (MSE): 3113.532 \n#> R squared (OOB): 0.355469\n\n\nUse this for prediction on new data, like for deploying" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "title": "2 - Feature Engineering", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 unique companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor (in the extra materials)" + "objectID": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-5", + "href": "archive/2022-08-Reykjavik-City/04-evaluating-models.html#your-turn-5", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nEnd of the day discussion!\nWhich model do you think you would decide to use?\nWhat surprised you the most?\nWhat is one thing you are looking forward to for tomorrow?\n\n\n\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#per-agent-statistics", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#per-agent-statistics", - "title": "2 - Feature Engineering", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-parameters", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-parameters", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#collapsing-factor-levels", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#collapsing-factor-levels", - "title": "2 - Feature Engineering", - "section": "Collapsing factor levels ", - "text": "Collapsing factor levels \nThere is a recipe step that will redefine factor levels based on their frequency in the training set:\n\nhotel_other_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_other(agent, threshold = 0.001) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nUsing this code, 34 agents (out of 98) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#optimize-tuning-parameters", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#optimize-tuning-parameters", + "title": "6 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#does-othering-help", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#does-othering-help", - "title": "2 - Feature Engineering", - "section": "Does othering help? ", - "text": "Does othering help? \n\nhotel_other_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hotel_other_rec)\n\nhotel_other_res <-\n hotel_other_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.213 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00341 Preprocessor1_Model1\n\nAbout the same MAE and much faster to complete.\nNow let’s look at a more sophisticated tool called effect feature hashing." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "title": "6 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing", - "title": "2 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nBetween agent and company, simple dummy variables would create 198 new columns (that are mostly zeros).\nAnother option is to have a binary indicator that combines some levels of these variables.\nFeature hashing (for more see FES, SMLTAR, TMwR, and aml4td):\n\nuses the character values of the levels\nconverts them to integer hash values\nuses the integers to assign them to a specific indicator column." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choosing-tuning-parameters", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choosing-tuning-parameters", + "title": "6 - Tuning Hyperparameters", + "section": "Choosing tuning parameters ", + "text": "Choosing tuning parameters \nLet’s take our previous recipe and add a few changes:\n\nglm_rec <-\n recipe(on_goal ~ ., data = nhl_train) %>%\n step_lencode_mixed(shooter, goaltender, outcome = vars(on_goal)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_mutate(\n angle = abs( atan2(abs(coord_y), (89 - coord_x) ) * (180 / pi) ),\n defensive_zone = ifelse(coord_x <= -25.5, 1, 0),\n behind_goal_line = ifelse(coord_x >= 89, 1, 0)\n ) %>%\n step_zv(all_predictors()) %>%\n step_ns(angle, deg_free = tune(\"angle\")) %>%\n step_ns(coord_x, deg_free = tune(\"coord_x\")) %>%\n step_normalize(all_numeric_predictors())\n\n\nLet’s tune() our spline terms!" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-1", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-1", - "title": "2 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nSuppose we want to use 32 indicator variables for agent.\nFor a agent with value “Max_Kuhn”, a hashing function converts it to an integer (say 210397726).\nTo assign it to one of the 32 columns, we would use modular arithmetic to assign it to a column:\n\n# For \"Max_Kuhn\" put a '1' in column: \n210397726 %% 32\n#> [1] 30\n\nHash functions are meant to emulate randomness." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choosing-tuning-parameters-1", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choosing-tuning-parameters-1", + "title": "6 - Tuning Hyperparameters", + "section": "Choosing tuning parameters ", + "text": "Choosing tuning parameters \nLet’s take our previous recipe and add a few changes:\n\nglm_spline_wflow <-\n workflow() %>%\n add_model(logistic_reg()) %>%\n add_recipe(glm_rec)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-pros", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-pros", - "title": "2 - Feature Engineering", - "section": "Feature Hashing Pros", - "text": "Feature Hashing Pros\n\nThe procedure will automatically work on new values of the predictors.\nIt is fast.\n“Signed” hashes add a sign to help avoid aliasing." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#section", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#section", + "title": "6 - Tuning Hyperparameters", + "section": "", + "text": "Splines replace the existing numeric predictor with a set of columns that allow a model to emulate a flexible, nonlinear relationship.\nMore spline terms = more “wiggly”, i.e. flexibly model a nonlinear relationship\nHow many spline terms? This is called degrees of freedom\n2 and 5 look like they underfit; 20 and 100 look like they overfit" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-cons", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-cons", - "title": "2 - Feature Engineering", - "section": "Feature Hashing Cons", - "text": "Feature Hashing Cons\n\nThere is no real logic behind which factor levels are combined.\nWe don’t know how many columns to add (more in the next section).\nSome columns may have all zeros.\nIf a indicator column is important to the model, we can’t easily determine why.\n\n\nThe signed hash make it slightly more possible to differentiate between confounded levels" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#splines-and-nonlinear-relationships", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#splines-and-nonlinear-relationships", + "title": "6 - Tuning Hyperparameters", + "section": "Splines and nonlinear relationships", + "text": "Splines and nonlinear relationships\n\n\nOur hockey data exhibits nonlinear relationships\nWe can model nonlinearity like this via a model (later this afternoon) or feature engineering\nHow do we decide how “wiggly” or flexible to make our spline features? TUNING" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes", - "title": "2 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \nThe textrecipes package has a step that can be added to the recipe:\n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nhotel_hash_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hash_rec)" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#grid-search", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#grid-search", + "title": "6 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nParameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", - "title": "2 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \n\nhotel_hash_res <-\n hotel_hash_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_hash_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.239 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00324 Preprocessor1_Model1\n\nAbout the same performance but now we can handle new values." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid", + "title": "6 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\nglm_spline_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 2 parameters for tuning\n#> \n#> identifier type object\n#> angle deg_free nparam[+]\n#> coord_x deg_free nparam[+]\n\n\nA parameter set can be updated (e.g. to change the ranges)." }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#debugging-a-recipe", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#debugging-a-recipe", - "title": "2 - Feature Engineering", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. hash_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid-1", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid-1", + "title": "6 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\n\n\nset.seed(12)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 2\n#> angle coord_x\n#> <int> <int>\n#> 1 5 8\n#> 2 1 13\n#> 3 8 6\n#> 4 11 5\n#> 5 4 1\n#> 6 6 10\n#> 7 10 15\n#> 8 12 13\n#> 9 8 13\n#> 10 10 6\n#> # ℹ 15 more rows\n\n\n\n\nA space-filling design like this tends to perform better than random grids.\nSpace-filling designs are also usually more efficient than regular grids." }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#example", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#example", - "title": "2 - Feature Engineering", - "section": "Example ", - "text": "Example \n\nhash_rec_fit <- prep(hash_rec)\n\n# Get the transformation coefficient\ntidy(hash_rec_fit, number = 1)\n\n# Get the processed data\nbake(hash_rec_fit, hotel_train %>% slice(1:3), contains(\"_agent_\"))" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n03:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#more-on-recipes", - "href": "archive/2024-08-posit-conf/advanced-02-feature-engineering.html#more-on-recipes", - "title": "2 - Feature Engineering", - "section": "More on recipes", - "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid-2", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#create-a-grid-2", + "title": "6 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\nset.seed(12)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 25)\n\ngrid\n#> # A tibble: 225 × 2\n#> angle coord_x\n#> <int> <int>\n#> 1 1 1\n#> 2 2 1\n#> 3 3 1\n#> 4 4 1\n#> 5 5 1\n#> 6 6 1\n#> 7 7 1\n#> 8 8 1\n#> 9 9 1\n#> 10 10 1\n#> # ℹ 215 more rows\n\n\nNote that even though we requested 25x25=625 rows, we only got 15x15=225 back, since the deg_free parameters only have a range of 1->15." }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---setup", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---setup", - "title": "4 - Grid Search via Racing", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#update-parameter-ranges", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#update-parameter-ranges", + "title": "6 - Tuning Hyperparameters", + "section": "Update parameter ranges ", + "text": "Update parameter ranges \n\nset.seed(12)\ngrid <- \n glm_spline_wflow %>% \n extract_parameter_set_dials() %>% \n update(angle = spline_degree(c(2L, 50L)),\n coord_x = spline_degree(c(2L, 50L))) %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 2\n#> angle coord_x\n#> <int> <int>\n#> 1 14 27\n#> 2 4 42\n#> 3 26 20\n#> 4 36 16\n#> 5 13 3\n#> 6 20 33\n#> 7 31 49\n#> 8 40 44\n#> 9 24 45\n#> 10 34 18\n#> # ℹ 15 more rows\n\n\nEven though angle is a deg_free parameter in step_ns(), we don’t use the dials deg_free() object here. We have a special spline_degree() function that has better defaults for splines." }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---data-usage", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---data-usage", - "title": "4 - Grid Search via Racing", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#the-results", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#the-results", + "title": "6 - Tuning Hyperparameters", + "section": "The results ", + "text": "The results \n\n\ngrid %>% \n ggplot(aes(angle, coord_x)) +\n geom_point(size = 4)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---boosting-model", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#previously---boosting-model", - "title": "4 - Grid Search via Racing", - "section": "Previously - Boosting Model ", - "text": "Previously - Boosting Model \n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#spline-grid-search", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#spline-grid-search", + "title": "6 - Tuning Hyperparameters", + "section": "Spline grid search ", + "text": "Spline grid search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE, parallel_over = \"everything\")\n\nglm_spline_res <-\n glm_spline_wflow %>%\n tune_grid(resamples = nhl_val, grid = grid, control = ctrl)\n\nglm_spline_res\n#> # Tuning results\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [5348/1338]> validation <tibble [50 × 6]> <tibble [2 × 3]> <tibble [33,450 × 8]>\n#> \n#> There were issues with some computations:\n#> \n#> - Warning(s) x1: glm.fit: algorithm did not converge, glm.fit: fitted probabilities numerically 0 or 1 occurred\n#> - Warning(s) x1: glm.fit: fitted probabilities numerically 0 or 1 occurred\n#> \n#> Run `show_notes(.Last.tune.result)` for more information.\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#first-a-shameless-promotion", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#first-a-shameless-promotion", - "title": "4 - Grid Search via Racing", - "section": "First, a shameless promotion", - "text": "First, a shameless promotion" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-1", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-1", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nTune our glm_wflow.\nWhat happens if you don’t supply a grid argument to tune_grid()?\n\n\n\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#making-grid-search-more-efficient", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#making-grid-search-more-efficient", - "title": "4 - Grid Search via Racing", - "section": "Making Grid Search More Efficient", - "text": "Making Grid Search More Efficient\nIn the last section, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\nAlso, for some models, we can fit far fewer models than the number that are being evaluated.\n\nFor boosting, a model with X trees can often predict on candidates with less than X trees.\n\nBoth of these methods can lead to enormous speed-ups." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#grid-results", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#grid-results", + "title": "6 - Tuning Hyperparameters", + "section": "Grid results ", + "text": "Grid results \n\nautoplot(glm_spline_res)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#model-racing", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#model-racing", - "title": "4 - Grid Search via Racing", - "section": "Model Racing", - "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models but only for a few resamples.\nDetermine which candidates have a low probability of being selected.\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain)\n\nThis can result in fitting a small number of models." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(glm_spline_res)\n#> # A tibble: 50 × 8\n#> angle coord_x .metric .estimator mean n std_err .config \n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 14 27 accuracy binary 0.804 1 NA Preprocessor01_Model1\n#> 2 14 27 roc_auc binary 0.815 1 NA Preprocessor01_Model1\n#> 3 4 42 accuracy binary 0.808 1 NA Preprocessor02_Model1\n#> 4 4 42 roc_auc binary 0.820 1 NA Preprocessor02_Model1\n#> 5 26 20 accuracy binary 0.805 1 NA Preprocessor03_Model1\n#> 6 26 20 roc_auc binary 0.819 1 NA Preprocessor03_Model1\n#> 7 36 16 accuracy binary 0.800 1 NA Preprocessor04_Model1\n#> 8 36 16 roc_auc binary 0.817 1 NA Preprocessor04_Model1\n#> 9 13 3 accuracy binary 0.803 1 NA Preprocessor05_Model1\n#> 10 13 3 roc_auc binary 0.807 1 NA Preprocessor05_Model1\n#> # ℹ 40 more rows" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#discarding-candidates", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#discarding-candidates", - "title": "4 - Grid Search via Racing", - "section": "Discarding Candidates", - "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large difference between resamples in the results." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-1", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-1", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(glm_spline_res, summarize = FALSE)\n#> # A tibble: 50 × 7\n#> id angle coord_x .metric .estimator .estimate .config \n#> <chr> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 validation 14 27 accuracy binary 0.804 Preprocessor01_Model1\n#> 2 validation 14 27 roc_auc binary 0.815 Preprocessor01_Model1\n#> 3 validation 4 42 accuracy binary 0.808 Preprocessor02_Model1\n#> 4 validation 4 42 roc_auc binary 0.820 Preprocessor02_Model1\n#> 5 validation 26 20 accuracy binary 0.805 Preprocessor03_Model1\n#> 6 validation 26 20 roc_auc binary 0.819 Preprocessor03_Model1\n#> 7 validation 36 16 accuracy binary 0.800 Preprocessor04_Model1\n#> 8 validation 36 16 roc_auc binary 0.817 Preprocessor04_Model1\n#> 9 validation 13 3 accuracy binary 0.803 Preprocessor05_Model1\n#> 10 validation 13 3 roc_auc binary 0.807 Preprocessor05_Model1\n#> # ℹ 40 more rows" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#resampling-results-non-racing", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#resampling-results-non-racing", - "title": "4 - Grid Search via Racing", - "section": "Resampling Results (Non-Racing)", - "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choose-a-parameter-combination", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choose-a-parameter-combination", + "title": "6 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \n\nshow_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 8\n#> angle coord_x .metric .estimator mean n std_err .config \n#> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 4 42 roc_auc binary 0.820 1 NA Preprocessor02_Model1\n#> 2 26 20 roc_auc binary 0.819 1 NA Preprocessor03_Model1\n#> 3 36 16 roc_auc binary 0.817 1 NA Preprocessor04_Model1\n#> 4 40 44 roc_auc binary 0.817 1 NA Preprocessor08_Model1\n#> 5 37 27 roc_auc binary 0.816 1 NA Preprocessor15_Model1" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#are-candidates-different", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#are-candidates-different", - "title": "4 - Grid Search via Racing", - "section": "Are Candidates Different?", - "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval for the difference in RMSE is (0.99, 2.8), indicating that candidate number 2 has smaller error." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "title": "6 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nselect_best(glm_spline_res, metric = \"roc_auc\")\n#> # A tibble: 1 × 3\n#> angle coord_x .config \n#> <int> <int> <chr> \n#> 1 4 42 Preprocessor02_Model1\n\n\nThis best result has:\n\nlow-degree spline for angle (less “wiggly”, less complex)\nhigher-degree spline for coord_x (more “wiggly”, more complex)" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#evaluating-differences-in-candidates", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#evaluating-differences-in-candidates", - "title": "4 - Grid Search via Racing", - "section": "Evaluating Differences in Candidates", - "text": "Evaluating Differences in Candidates\n\n\nWhat if we were to have compared the candidates while we seqeuntially evaluated each resample?\n👉\n\nOne candidate shows superiority when 4 resamples have been evaluated." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-trees-1", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-trees-1", + "title": "6 - Tuning Hyperparameters", + "section": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\n\n\nReview how a decision tree model works:\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#interim-analysis-of-results", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#interim-analysis-of-results", - "title": "4 - Grid Search via Racing", - "section": "Interim Analysis of Results", - "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order (so set the seed).\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#single-decision-tree", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#single-decision-tree", + "title": "6 - Tuning Hyperparameters", + "section": "Single decision tree", + "text": "Single decision tree" }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#racing", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#racing", - "title": "4 - Grid Search via Racing", - "section": "Racing ", - "text": "Racing \n\n# Let's use a larger grid\nset.seed(8945)\nlgbm_grid <- \n lgbm_param %>% \n grid_space_filling(size = 50)\n\nlibrary(finetune)\n\nset.seed(9)\nlgbm_race_res <-\n lgbm_wflow %>%\n tune_race_anova(\n resamples = hotel_rs,\n grid = lgbm_grid, \n metrics = reg_metrics\n )\n\nThe syntax and helper functions are extremely similar to those shown for tune_grid()." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-trees-2", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-trees-2", + "title": "6 - Tuning Hyperparameters", + "section": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Boosted trees 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\nBoosting methods fit a sequence of tree-based models.\n\n\nEach tree is dependent on the one before and tries to compensate for any poor results in the previous trees.\nThis is like gradient-based steepest ascent methods from calculus." }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#racing-results", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#racing-results", - "title": "4 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\nshow_best(lgbm_race_res, metric = \"mae\")\n#> # A tibble: 2 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1347 5 0.0655 66 26 mae standard 9.64 10 0.173 Preprocessor34_Model1\n#> 2 980 8 0.0429 17 135 mae standard 9.76 10 0.164 Preprocessor25_Model1" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "title": "6 - Tuning Hyperparameters", + "section": "Boosted tree tuning parameters", + "text": "Boosted tree tuning parameters\nMost modern boosting methods have a lot of tuning parameters!\n\n\nFor tree growth and pruning (min_n, max_depth, etc)\nFor boosting (trees, stop_iter, learn_rate)\n\n\n\nWe’ll use early stopping to stop boosting when a few iterations produce consecutively worse results." }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#racing-results-1", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#racing-results-1", - "title": "4 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\n\nOnly 171 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_race_res) + \n scale_x_continuous(breaks = pretty_breaks())" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#comparing-tree-ensembles", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#comparing-tree-ensembles", + "title": "6 - Tuning Hyperparameters", + "section": "Comparing tree ensembles", + "text": "Comparing tree ensembles\n\n\nRandom forest\n\nIndependent trees\nBootstrapped data\nNo pruning\n1000’s of trees\n\n\nBoosting\n\nDependent trees\nDifferent case weights\nTune tree parameters\nFar fewer trees\n\n\nThe general consensus for tree-based models is, in terms of performance: boosting > random forest > bagging > single trees." }, { - "objectID": "archive/2024-08-posit-conf/advanced-04-racing.html#your-turn", - "href": "archive/2024-08-posit-conf/advanced-04-racing.html#your-turn", - "title": "4 - Grid Search via Racing", - "section": "Your turn", - "text": "Your turn\n\nRun tune_race_anova() with a different seed.\nDid you get the same or similar results?\n\n\n\n\n−+\n10:00" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-tree-code", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#boosted-tree-code", + "title": "6 - Tuning Hyperparameters", + "section": "Boosted tree code", + "text": "Boosted tree code\n\nxgb_spec <-\n boost_tree(\n trees = tune(), min_n = tune(), tree_depth = tune(),\n learn_rate = tune(), loss_reduction = tune()\n ) %>%\n set_mode(\"classification\") %>% \n set_engine(\"xgboost\") \n\nxgb_rec <- \n recipe(on_goal ~ ., data = nhl_train) %>% \n step_lencode_mixed(shooter, goaltender, outcome = vars(on_goal)) %>% \n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\nxgb_wflow <- \n workflow() %>% \n add_model(xgb_spec) %>% \n add_recipe(xgb_rec)\n\n\nvalidation is an argument to parsnip::xgb_train(), not directly to xgboost. It generates a validation set that is used by xgboost when evaluating model performance. It is eventually assigned to xgb.train(watchlist = list(validation = data)).\nSee translate(xgb_spec) to see where it is passed to parsnip::xgb_train()." }, { - "objectID": "archive/2024-08-posit-conf/advanced-06-wrapping-up.html#your-turn", - "href": "archive/2024-08-posit-conf/advanced-06-wrapping-up.html#your-turn", - "title": "6 - Wrapping up", + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-2", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-2", + "title": "6 - Tuning Hyperparameters", "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" - }, - { - "objectID": "archive/2024-08-posit-conf/advanced-06-wrapping-up.html#resources-to-keep-learning", - "href": "archive/2024-08-posit-conf/advanced-06-wrapping-up.html#resources-to-keep-learning", - "title": "6 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" + "text": "Your turn\n\nCreate your boosted tree workflow.\n\n\n\n03:00" }, { - "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#previously---setup", - "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#previously---setup", - "title": "Extras - Effect Encodings", - "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#running-in-parallel", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#running-in-parallel", + "title": "6 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" }, { - "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#previously---data-usage", - "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#previously---data-usage", - "title": "Extras - Effect Encodings", - "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#running-in-parallel-1", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#running-in-parallel-1", + "title": "6 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." }, { - "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "title": "Extras - Effect Encodings", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning ", + "text": "Tuning \nThis will take some time to run ⏳\n\nset.seed(9)\n\nxgb_res <-\n xgb_wflow %>%\n tune_grid(resamples = nhl_val, grid = 30, control = ctrl) # automatic grid now!" }, { - "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#per-agent-statistics", - "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#per-agent-statistics", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-3", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-3", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nStart tuning the boosted tree model!\nWe won’t wait for everyone’s tuning to finish, but take this time to get it started before we move on.\n\n\n\n03:00" }, { - "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#what-is-an-effect-encoding", - "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#what-is-an-effect-encoding", - "title": "Extras - Effect Encodings", - "section": "What is an effect encoding?", - "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <fct> <int>\n#> 1 52.7 cynthia_worsley 1\n#> 2 51.8 carlos_bryant 2\n#> 3 53.8 lance_hitchcock 3\n#> 4 51.8 lance_hitchcock 4\n#> 5 46.8 cynthia_worsley 5\n#> 6 54.7 charles_najera 6\n#> 7 46.8 cynthia_worsley 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <dbl> <int>\n#> 1 52.7 88.5 1\n#> 2 51.8 89.5 2\n#> 3 53.8 79.8 3\n#> 4 51.8 79.8 4\n#> 5 46.8 88.5 5\n#> 6 54.7 109. 6\n#> 7 46.8 88.5 7\n\n\nThe agent column is replaced with an estimate of the ADR." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-2", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-2", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\nxgb_res\n#> # Tuning results\n#> # Validation Set Split (0.8/0.2) \n#> # A tibble: 1 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [5348/1338]> validation <tibble [60 × 9]> <tibble [0 × 3]> <tibble [40,140 × 11]>" }, { - "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#per-agent-statistics-again", - "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#per-agent-statistics-again", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics again", - "text": "Per-agent statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these means use partial pooling.\nPooling borrows strength across agents and shrinks extreme values towards the mean for agents with very few transations\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for agents with fewer reservations by shrinking the estimate to the overall ADR mean" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-3", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#tuning-results-3", + "title": "6 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\nautoplot(xgb_res)" }, { - "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#partial-pooling", - "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#partial-pooling", - "title": "Extras - Effect Encodings", - "section": "Partial pooling", - "text": "Partial pooling" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#again-with-the-location-features", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#again-with-the-location-features", + "title": "6 - Tuning Hyperparameters", + "section": "Again with the location features", + "text": "Again with the location features\n\ncoord_rec <- \n xgb_rec %>%\n step_mutate(\n angle = abs( atan2(abs(coord_y), (89 - coord_x) ) * (180 / pi) ),\n defensive_zone = ifelse(coord_x <= -25.5, 1, 0),\n behind_goal_line = ifelse(coord_x >= 89, 1, 0)\n )\n\nxgb_coord_wflow <- \n workflow() %>% \n add_model(xgb_spec) %>% \n add_recipe(coord_rec)\n\nset.seed(9)\nxgb_coord_res <-\n xgb_coord_wflow %>%\n tune_grid(resamples = nhl_val, grid = 30, control = ctrl)" }, { - "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#agent-effects", - "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#agent-effects", - "title": "Extras - Effect Encodings", - "section": "Agent effects ", - "text": "Agent effects \n\nlibrary(embed)\n\nhotel_effect_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>% \n step_YeoJohnson(lead_time) %>%\n step_lencode_mixed(agent, company, outcome = vars(avg_price_per_room)) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#did-the-machine-figure-it-out", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#did-the-machine-figure-it-out", + "title": "6 - Tuning Hyperparameters", + "section": "Did the machine figure it out?", + "text": "Did the machine figure it out?\nNo extra features:\n\nshow_best(xgb_res, metric = \"roc_auc\", n = 3)\n#> # A tibble: 3 × 11\n#> trees min_n tree_depth learn_rate loss_reduction .metric .estimator mean n std_err .config \n#> <int> <int> <int> <dbl> <dbl> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1697 3 3 0.0116 0.674 roc_auc binary 0.804 1 NA Preprocessor1_Model02\n#> 2 1264 16 2 0.00732 0.000340 roc_auc binary 0.803 1 NA Preprocessor1_Model12\n#> 3 569 39 4 0.0272 0.0000288 roc_auc binary 0.800 1 NA Preprocessor1_Model30\n\nWith additional coordinate features:\n\nshow_best(xgb_coord_res, metric = \"roc_auc\", n = 3)\n#> # A tibble: 3 × 11\n#> trees min_n tree_depth learn_rate loss_reduction .metric .estimator mean n std_err .config \n#> <int> <int> <int> <dbl> <dbl> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1697 3 3 0.0116 0.674 roc_auc binary 0.807 1 NA Preprocessor1_Model02\n#> 2 1264 16 2 0.00732 0.000340 roc_auc binary 0.804 1 NA Preprocessor1_Model12\n#> 3 427 31 2 0.249 0.0000000773 roc_auc binary 0.803 1 NA Preprocessor1_Model23" }, { - "objectID": "archive/2024-08-posit-conf/extras-effect-encodings.html#effect-encoding-results", - "href": "archive/2024-08-posit-conf/extras-effect-encodings.html#effect-encoding-results", - "title": "Extras - Effect Encodings", - "section": "Effect encoding results ", - "text": "Effect encoding results \n\nhotel_effect_wflow <-\n workflow() %>%\n add_model(linear_reg()) %>% \n update_recipe(hotel_effect_rec)\n\nreg_metrics <- metric_set(mae, rsq)\n\nhotel_effect_res <-\n hotel_effect_wflow %>%\n fit_resamples(hotel_rs, metrics = reg_metrics)\n\ncollect_metrics(hotel_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.8 10 0.189 Preprocessor1_Model1\n#> 2 rsq standard 0.870 10 0.00357 Preprocessor1_Model1\n\nSlightly worse but it can handle new agents (if they occur)." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#compare-models", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#compare-models", + "title": "6 - Tuning Hyperparameters", + "section": "Compare models", + "text": "Compare models\nBest logistic regression results:\n\nglm_spline_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, .estimator, mean, n, std_err, .config)\n#> # A tibble: 1 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 roc_auc binary 0.820 1 NA Preprocessor02_Model1\n\n\nBest boosting results:\n\nxgb_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, .estimator, mean, n, std_err, .config)\n#> # A tibble: 1 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 roc_auc binary 0.804 1 NA Preprocessor1_Model02" }, { - "objectID": "archive/2024-08-posit-conf/extras-vetiver.html#deploying-a-model", - "href": "archive/2024-08-posit-conf/extras-vetiver.html#deploying-a-model", - "title": "Extras - Model deployment", - "section": "Deploying a model ", - "text": "Deploying a model \nWe have a decision tree, tree_fit, to model whether or not a plot of land in Washington is forested or not.\nHow do we use our model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"forested\")\nv\n#> \n#> ── forested ─ <bundled_workflow> model for deployment \n#> A rpart classification modeling workflow using 18 features\n\nLearn more at https://vetiver.rstudio.com" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#updating-the-workflow", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#updating-the-workflow", + "title": "6 - Tuning Hyperparameters", + "section": "Updating the workflow ", + "text": "Updating the workflow \n\nbest_auc <- select_best(glm_spline_res, metric = \"roc_auc\")\nbest_auc\n#> # A tibble: 1 × 3\n#> angle coord_x .config \n#> <int> <int> <chr> \n#> 1 4 42 Preprocessor02_Model1\n\nglm_spline_wflow <-\n glm_spline_wflow %>% \n finalize_workflow(best_auc)\n\nglm_spline_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: logistic_reg()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 7 Recipe Steps\n#> \n#> • step_lencode_mixed()\n#> • step_dummy()\n#> • step_mutate()\n#> • step_zv()\n#> • step_ns()\n#> • step_ns()\n#> • step_normalize()\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm" }, { - "objectID": "archive/2024-08-posit-conf/extras-vetiver.html#deploy-your-model-1", - "href": "archive/2024-08-posit-conf/extras-vetiver.html#deploy-your-model-1", - "title": "Extras - Model deployment", - "section": "Deploy your model ", - "text": "Deploy your model \nHow do we use our model in production?\n\nlibrary(plumber)\npr() %>%\n vetiver_api(v)\n#> # Plumber router with 4 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Users/simoncouch/Library/R/arm64/4.4/library/vetiver\n#> ├──/metadata (GET)\n#> ├──/ping (GET)\n#> ├──/predict (POST)\n#> └──/prototype (GET)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#the-final-fit-to-the-nhl-data", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#the-final-fit-to-the-nhl-data", + "title": "6 - Tuning Hyperparameters", + "section": "The final fit to the NHL data ", + "text": "The final fit to the NHL data \n\ntest_res <- \n glm_spline_wflow %>% \n last_fit(split = nhl_split)\n\ntest_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [6686/2229]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [2,229 × 6]> <workflow>\n\n\nRemember that last_fit() fits one time with the combined training and validation set, then evaluates one time with the testing set." }, { - "objectID": "archive/2024-08-posit-conf/extras-vetiver.html#your-turn", - "href": "archive/2024-08-posit-conf/extras-vetiver.html#your-turn", - "title": "Extras - Model deployment", + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-4", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-4", + "title": "6 - Tuning Hyperparameters", "section": "Your turn", - "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n−+\n05:00" + "text": "Your turn\n\nFinalize your workflow with the best parameters.\nCreate a final fit.\n\n\n\n08:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#venue-information", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#venue-information", - "title": "1 - Introduction", - "section": "Venue information", - "text": "Venue information\n\nThere are gender neutral bathrooms located on levels 3, 4, 5, 6 & 7\nA meditation/prayer room is located in 503\n(Mon & Tue 7am - 7pm, and Wed 7am - 5pm)\nA lactation room is located in 509\n(Mon & Tue 7am - 7pm, and Wed 7am - 5pm)" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#estimates-of-roc-auc", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#estimates-of-roc-auc", + "title": "6 - Tuning Hyperparameters", + "section": "Estimates of ROC AUC ", + "text": "Estimates of ROC AUC \nValidation results from tuning:\n\nglm_spline_res %>% \n show_best(metric = \"roc_auc\", n = 1) %>% \n select(.metric, mean, n, std_err)\n#> # A tibble: 1 × 4\n#> .metric mean n std_err\n#> <chr> <dbl> <int> <dbl>\n#> 1 roc_auc 0.820 1 NA\n\n\nTest set results:\n\ntest_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.800 Preprocessor1_Model1\n#> 2 roc_auc binary 0.807 Preprocessor1_Model1" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#workshop-policies", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#workshop-policies", - "title": "1 - Introduction", - "section": "Workshop policies", - "text": "Workshop policies\n\nPlease review the posit::conf code of conduct, which applies to all workshops: https://posit.co/code-of-conduct\nCoC site has info on how to report a problem (in person, email, phone)\nPlease do not photograph people wearing red lanyards" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#final-fitted-workflow", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#final-fitted-workflow", + "title": "6 - Tuning Hyperparameters", + "section": "Final fitted workflow", + "text": "Final fitted workflow\nExtract the final fitted workflow, fit using the training set:\n\nfinal_glm_spline_wflow <- \n test_res %>% \n extract_workflow()\n\n# use this object to predict or deploy\npredict(final_glm_spline_wflow, nhl_test[1:3,])\n#> # A tibble: 3 × 1\n#> .pred_class\n#> <fct> \n#> 1 yes \n#> 2 yes \n#> 3 no" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#who-are-you", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have some exposure to basic statistical concepts like linear models and residuals\nYou do not need intermediate or expert familiarity with modeling or ML" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#next-steps", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#next-steps", + "title": "6 - Tuning Hyperparameters", + "section": "Next steps", + "text": "Next steps\n\nDocument the model.\n\n\n\nDeploy the model.\n\n\n\n\nCreate an applicability domain model to help monitor our data over time.\n\n\n\n\nUse explainers to characterize the model and the predictions." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#who-are-tidymodels", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#who-are-tidymodels", - "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\n+ our TA today, Sara Altman!\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-yourself", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-yourself", + "title": "6 - Tuning Hyperparameters", + "section": "Explain yourself", + "text": "Explain yourself\nThere are two categories of model explanations, global and local.\n\n\nGlobal model explanations provide an overall understanding aggregated over a whole set of observations.\nLocal model explanations provide information about a prediction for a single observation.\n\n\n\nYou can also build global model explanations by aggregating local model explanations." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#asking-for-help", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#asking-for-help", - "title": "1 - Introduction", - "section": "Asking for help", - "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#a-tidymodels-explainer", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#a-tidymodels-explainer", + "title": "6 - Tuning Hyperparameters", + "section": "A tidymodels explainer", + "text": "A tidymodels explainer\nWe can build explainers using:\n\noriginal, basic predictors\nderived features\n\n\nlibrary(DALEXtra)\n\nglm_explainer <- explain_tidymodels(\n final_glm_spline_wflow,\n data = dplyr::select(nhl_train, -on_goal),\n # DALEX required an integer for factors:\n y = as.integer(nhl_train$on_goal) - 1,\n verbose = FALSE\n)" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#discord", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#discord", - "title": "1 - Introduction", - "section": "Discord ", - "text": "Discord \n\npos.it/conf-event-portal (login)\nClick on “Join Discord, the virtual networking platform!”\nBrowse Channels -> #workshop-tidymodels-intro" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates", + "title": "6 - Tuning Hyperparameters", + "section": "Explain the x coordinates", + "text": "Explain the x coordinates\nWith our explainer, let’s create partial dependence profiles:\n\nset.seed(123)\npdp_coord_x <- model_profile(\n glm_explainer,\n variables = \"coord_x\",\n N = 500,\n groups = \"strength\"\n)\n\n\nYou can use the default plot() method or create your own visualization." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#section-2", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#section-2", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates-1", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates-1", + "title": "6 - Tuning Hyperparameters", + "section": "Explain the x coordinates", + "text": "Explain the x coordinates" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#section-3", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#section-3", - "title": "1 - Introduction", - "section": "👀", - "text": "👀\n\n\nhttps://workshops.tidymodels.org" + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates-2", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#explain-the-x-coordinates-2", + "title": "6 - Tuning Hyperparameters", + "section": "Explain the x coordinates", + "text": "Explain the x coordinates" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#plan-for-this-workshop", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Plan for this workshop", - "text": "Plan for this workshop\n\nYour data budget\nWhat makes a model\nEvaluating models\nTuning models\n\n\nThis workshop will well-prepare folks going on to the Advanced tidymodels workshop, which will cover feature engineering and much more on hyperparameter tuning." + "objectID": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-5", + "href": "archive/2022-08-Reykjavik-City/06-tuning-hyperparameters.html#your-turn-5", + "title": "6 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nCreate an explainer for our glm model.\nTry using other variables, like extra_attacker or game_seconds.\n\n\n\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#section-4", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#section-4", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free): TODO-ADD-LATER" + "objectID": "archive/2022-08-Reykjavik-City/08-wrapping-up.html#your-turn", + "href": "archive/2022-08-Reykjavik-City/08-wrapping-up.html#your-turn", + "title": "8 - Wrapping up", + "section": "Your turn", + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" + "objectID": "archive/2022-08-Reykjavik-City/08-wrapping-up.html#resources-to-keep-learning", + "href": "archive/2022-08-Reykjavik-City/08-wrapping-up.html#resources-to-keep-learning", + "title": "8 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Twitter and at the tidyverse blog for updates!" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-1", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-1", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "archive/2022-08-Reykjavik-City/index.html", + "href": "archive/2022-08-Reykjavik-City/index.html", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels presented in Reykjavík Iceland. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-2024-edition", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-2024-edition", - "title": "1 - Introduction", - "section": "What is machine learning? (2024 edition)", - "text": "What is machine learning? (2024 edition)\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/\n\n\nIn the early 2010s, “Artificial intelligence” (AI) was largely synonymous with what we’ll refer to as “machine learning” in this workshop. In the late 2010s and early 2020s, AI usually referred to deep learning methods. Since the release of ChatGPT in late 2022, “AI” has come to also encompass large language models / generative models." + "objectID": "archive/2022-08-Reykjavik-City/index.html#welcome", + "href": "archive/2022-08-Reykjavik-City/index.html#welcome", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels presented in Reykjavík Iceland. This workshop provides an introduction to machine learning with R using the tidymodels framework, a collection of packages for modeling and machine learning using tidyverse principles. We will build, evaluate, compare, and tune predictive models. Along the way, we’ll learn about key concepts in machine learning including overfitting, resampling, and feature engineering. Learners will gain knowledge about good predictive modeling practices, as well as hands-on experience using tidymodels packages like parsnip, rsample, recipes, yardstick, tune, and workflows." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-2", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-machine-learning-2", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "archive/2022-08-Reykjavik-City/index.html#is-this-workshop-for-me", + "href": "archive/2022-08-Reykjavik-City/index.html#is-this-workshop-for-me", + "title": "Machine learning with tidymodels", + "section": "Is this workshop for me? ", + "text": "Is this workshop for me? \nThis course assumes intermediate R knowledge. This workshop is for you if:\n\nYou can use the magrittr pipe %>% and/or native pipe |>\nYou are familiar with functions from dplyr, tidyr, and ggplot2\nYou can read data into R, transform and reshape data, and make a wide variety of graphs\n\nWe expect participants to have some exposure to basic statistical concepts, but NOT intermediate or expert familiarity with modeling or machine learning." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#your-turn", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#your-turn", - "title": "1 - Introduction", - "section": "Your turn", - "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + "objectID": "archive/2022-08-Reykjavik-City/index.html#preparation", + "href": "archive/2022-08-Reykjavik-City/index.html#preparation", + "title": "Machine learning with tidymodels", + "section": "Preparation", + "text": "Preparation\nPlease join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2022.02), available at https://www.rstudio.com/download\nThe following R packages, which you can install from the R console:\n\n\ninstall.packages(c(\"Cubist\", \"DALEXtra\", \"doParallel\", \"earth\", \"embed\", \n \"forcats\", \"lme4\", \"parallelly\", \"ranger\", \"remotes\", \"rpart\", \n \"rpart.plot\", \"rules\", \"stacks\", \"tidymodels\",\n \"vetiver\", \"xgboost\"))\n\nremotes::install_github(\"topepo/ongoal@hockeyR\")" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-tidymodels", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#what-is-tidymodels", - "title": "1 - Introduction", - "section": "What is tidymodels? ", - "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.2.0 ──\n#> ✔ broom 1.0.6 ✔ rsample 1.2.1\n#> ✔ dials 1.3.0 ✔ tibble 3.2.1\n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1\n#> ✔ infer 1.0.7 ✔ tune 1.2.1\n#> ✔ modeldata 1.4.0 ✔ workflows 1.1.4\n#> ✔ parsnip 1.2.1 ✔ workflowsets 1.1.0\n#> ✔ purrr 1.0.2 ✔ yardstick 1.3.1\n#> ✔ recipes 1.1.0\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Use tidymodels_prefer() to resolve common conflicts." + "objectID": "archive/2022-08-Reykjavik-City/index.html#slides", + "href": "archive/2022-08-Reykjavik-City/index.html#slides", + "title": "Machine learning with tidymodels", + "section": "Slides", + "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. There are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nDay One\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n\n\n\nDay Two\n\n05: Feature engineering\n06: Tuning hyperparameters\n07: Transit Case Study\n08: Wrapping up\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\nRoadmap for today\nMinimal version of predictive modeling process\nFeature engineering and tuning as iterative extensions" + "objectID": "archive/2022-08-Reykjavik-City/index.html#code", + "href": "archive/2022-08-Reykjavik-City/index.html#code", + "title": "Machine learning with tidymodels", + "section": "Code", + "text": "Code\nQuarto files (version 1.4.104) for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" + }, + { + "objectID": "archive/2022-08-Reykjavik-City/index.html#past-workshops", + "href": "archive/2022-08-Reykjavik-City/index.html#past-workshops", + "title": "Machine learning with tidymodels", + "section": "Past workshops", + "text": "Past workshops\n\n25-26 July 2022 at rstudio::conf()\n16-17 August 2022 in Reykjavik Iceland" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-1", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-1", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2022-08-Reykjavik-City/index.html#acknowledgments", + "href": "archive/2022-08-Reykjavik-City/index.html#acknowledgments", + "title": "Machine learning with tidymodels", + "section": "Acknowledgments", + "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-2", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-2", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." + "objectID": "archive/2022-08-Reykjavik-City/index.html#reuse-and-licensing", + "href": "archive/2022-08-Reykjavik-City/index.html#reuse-and-licensing", + "title": "Machine learning with tidymodels", + "section": "Reuse and licensing", + "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-3", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-3", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips", + "href": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips", + "title": "2 - Your data budget", + "section": "Data on Chicago taxi trips", + "text": "Data on Chicago taxi trips\n\n\n\nThe city of Chicago releases anonymized trip-level data on taxi trips in the city.\nWe pulled a sample of 10,000 rides occurring in early 2022.\nType ?modeldatatoo::data_taxi() to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/8322/taxi" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-4", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-4", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-07-nyr/02-data-budget.html#which-of-these-variables-can-we-use", + "href": "archive/2023-07-nyr/02-data-budget.html#which-of-these-variables-can-we-use", + "title": "2 - Your data budget", + "section": "Which of these variables can we use?", + "text": "Which of these variables can we use?\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\n\ntaxi <- data_taxi()\n\nnames(taxi)\n#> [1] \"tip\" \"id\" \"duration\" \"distance\" \"fare\" \n#> [6] \"tolls\" \"extras\" \"total_cost\" \"payment_type\" \"company\" \n#> [11] \"local\" \"dow\" \"month\" \"hour\"" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-5", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-5", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-07-nyr/02-data-budget.html#checklist-for-predictors", + "href": "archive/2023-07-nyr/02-data-budget.html#checklist-for-predictors", + "title": "2 - Your data budget", + "section": "Checklist for predictors", + "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-6", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-6", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-1", + "href": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-1", + "title": "2 - Your data budget", + "section": "Data on Chicago taxi trips", + "text": "Data on Chicago taxi trips\nWe are using a slightly modified version from the modeldatatoo data.\n\ntaxi <- taxi %>%\n mutate(month = factor(month, levels = c(\"Jan\", \"Feb\", \"Mar\", \"Apr\"))) %>% \n select(-c(id, duration, fare, tolls, extras, total_cost, payment_type)) %>% \n drop_na()" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-7", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#the-whole-game-7", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-2", + "href": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-2", + "title": "2 - Your data budget", + "section": "Data on Chicago taxi trips", + "text": "Data on Chicago taxi trips\n\n\n\nN = 10,000\nA nominal outcome, tip, with levels \"yes\" and \"no\"\n6 other variables\n\ncompany, local, and dow, and month are nominal predictors\ndistance and hours are numeric predictors\n\n\n\n\n\n\nCredit: https://unsplash.com/photos/7_r85l4eht8\n\n\ntip: Whether the rider left a tip. A factor with levels “yes” and “no”.\ndistance: The trip distance, in odometer miles.\ncompany: The taxi company, as a factor. Companies that occurred few times were binned as “other”.\nlocal: Whether the trip started in the same community area as it began. See the source data for community area values.\ndow: The day of the week in which the trip began, as a factor.\nmonth: The month in which the trip began, as a factor.\nhour: The hour of the day in which the trip began, as a numeric." }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#lets-install-some-packages", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"forested\", \"lightgbm\", \"lme4\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)" + "objectID": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-3", + "href": "archive/2023-07-nyr/02-data-budget.html#data-on-chicago-taxi-trips-3", + "title": "2 - Your data budget", + "section": "Data on Chicago taxi trips", + "text": "Data on Chicago taxi trips\n\ntaxi\n#> # A tibble: 8,807 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 1.24 Sun Taxi no Thu Feb 13\n#> 2 no 5.39 Flash Cab no Sat Mar 12\n#> 3 yes 3.01 City Service no Wed Feb 17\n#> 4 no 18.4 Sun Taxi no Sat Apr 6\n#> 5 yes 1.76 Sun Taxi no Sun Jan 15\n#> 6 yes 13.6 Sun Taxi no Mon Feb 17\n#> 7 yes 3.71 City Service no Mon Mar 21\n#> 8 yes 4.8 other no Tue Mar 9\n#> 9 yes 18.0 City Service no Fri Jan 19\n#> 10 no 17.5 other yes Thu Apr 12\n#> # ℹ 8,797 more rows" }, { - "objectID": "archive/2024-08-posit-conf/intro-01-introduction.html#our-versions", - "href": "archive/2024-08-posit-conf/intro-01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nR version 4.4.1 (2024-06-14), Quarto (1.6.1)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.3.1\n\n\nbroom\n1.0.6\n\n\nCubist\n0.4.4\n\n\ndials\n1.3.0\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.3\n\n\nembed\n1.1.4\n\n\nfinetune\n1.2.0\n\n\nforested\n0.1.0\n\n\nFormula\n1.2-5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nggplot2\n3.5.1\n\n\nlattice\n0.22-6\n\n\nlightgbm\n4.5.0\n\n\nlme4\n1.1-35.5\n\n\nmodeldata\n1.4.0\n\n\nparallelly\n1.38.0\n\n\nparsnip\n1.2.1\n\n\nplotmo\n3.6.3\n\n\nplotrix\n3.8-4\n\n\nplumber\n1.2.2\n\n\nprobably\n1.0.3\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\npurrr\n1.0.2\n\n\nranger\n0.16.0\n\n\nrecipes\n1.1.0\n\n\nrpart\n4.1.23\n\n\nrpart.plot\n3.1.2\n\n\nrsample\n1.2.1\n\n\nrules\n1.0.2\n\n\nscales\n1.3.0\n\n\nsplines2\n0.5.3\n\n\nstacks\n1.0.5\n\n\ntext2vec\n0.6.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntextrecipes\n1.0.6\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.2.0\n\n\ntidyr\n1.3.1\n\n\ntune\n1.2.1\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.1.4\n\n\nworkflowsets\n1.1.0\n\n\nyardstick\n1.3.1" + "objectID": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending", + "href": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets\nbrulee for regression using torch" + "objectID": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-1", + "href": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-1", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-2", + "href": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-2", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-1", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-1", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" + "objectID": "archive/2023-07-nyr/02-data-budget.html#your-turn", + "href": "archive/2023-07-nyr/02-data-budget.html#your-turn", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-2", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-2", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2023-07-nyr/02-data-budget.html#the-initial-split", + "href": "archive/2023-07-nyr/02-data-budget.html#the-initial-split", + "title": "2 - Your data budget", + "section": "The initial split ", + "text": "The initial split \n\nset.seed(123)\ntaxi_split <- initial_split(taxi)\ntaxi_split\n#> <Training/Testing/Total>\n#> <6605/2202/8807>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-3", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-3", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" + "objectID": "archive/2023-07-nyr/02-data-budget.html#accessing-the-data", + "href": "archive/2023-07-nyr/02-data-budget.html#accessing-the-data", + "title": "2 - Your data budget", + "section": "Accessing the data ", + "text": "Accessing the data \n\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-4", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-4", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() %>%\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" + "objectID": "archive/2023-07-nyr/02-data-budget.html#the-training-set", + "href": "archive/2023-07-nyr/02-data-budget.html#the-training-set", + "title": "2 - Your data budget", + "section": "The training set", + "text": "The training set\n\ntaxi_train\n#> # A tibble: 6,605 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 4.54 City Service no Sat Mar 16\n#> 2 no 10.2 Flash Cab no Mon Feb 8\n#> 3 yes 12.4 other no Sun Apr 15\n#> 4 yes 15.3 Sun Taxi no Mon Apr 18\n#> 5 no 6.41 Flash Cab no Wed Apr 14\n#> 6 yes 1.56 other no Tue Jan 13\n#> 7 yes 3.13 Flash Cab no Sun Apr 12\n#> 8 yes 7.54 other no Tue Apr 8\n#> 9 yes 6.98 Flash Cab no Tue Apr 5\n#> 10 yes 0.7 Taxi Affiliation Services no Tue Jan 9\n#> # ℹ 6,595 more rows" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-5", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-5", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2023-07-nyr/02-data-budget.html#the-test-set", + "href": "archive/2023-07-nyr/02-data-budget.html#the-test-set", + "title": "2 - Your data budget", + "section": "The test set ", + "text": "The test set \n🙈\n\nThere are 2202 rows and 7 columns in the test set." }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-6", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-6", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" + "objectID": "archive/2023-07-nyr/02-data-budget.html#your-turn-1", + "href": "archive/2023-07-nyr/02-data-budget.html#your-turn-1", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-7", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-7", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree() %>% \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" + "objectID": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-3", + "href": "archive/2023-07-nyr/02-data-budget.html#data-splitting-and-spending-3", + "title": "2 - Your data budget", + "section": "Data splitting and spending ", + "text": "Data splitting and spending \n\nset.seed(123)\ntaxi_split <- initial_split(taxi, prop = 0.8)\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)\n\nnrow(taxi_train)\n#> [1] 7045\nnrow(taxi_test)\n#> [1] 1762" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-8", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#to-specify-a-model-8", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2023-07-nyr/02-data-budget.html#section-1", + "href": "archive/2023-07-nyr/02-data-budget.html#section-1", + "title": "2 - Your data budget", + "section": "", + "text": "We will use this tomorrow" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-1", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-1", - "title": "3 - What makes a model?", + "objectID": "archive/2023-07-nyr/02-data-budget.html#your-turn-2", + "href": "archive/2023-07-nyr/02-data-budget.html#your-turn-2", + "title": "2 - Your data budget", "section": "Your turn", - "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a logistic regression model.\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/\n\n\nExtension/Challenge: Edit this code to use a different model. For example, try using a conditional inference tree as implemented in the partykit package by changing the engine - or try an entirely different model type!\n\n\n\n−+\n05:00" + "text": "Your turn\n\nExplore the taxi_train data on your own!\n\nWhat’s the distribution of the outcome, tip?\nWhat’s the distribution of numeric variables like distance?\nHow does tip differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#models-well-be-using-today", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#models-well-be-using-today", - "title": "3 - What makes a model?", - "section": "Models we’ll be using today", - "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" + "objectID": "archive/2023-07-nyr/02-data-budget.html#section-3", + "href": "archive/2023-07-nyr/02-data-budget.html#section-3", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n ggplot(aes(x = tip)) +\n geom_bar()" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" + "objectID": "archive/2023-07-nyr/02-data-budget.html#section-4", + "href": "archive/2023-07-nyr/02-data-budget.html#section-4", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n ggplot(aes(x = tip, fill = local)) +\n geom_bar() +\n scale_fill_viridis_d(end = .5)" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-1", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-1", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" + "objectID": "archive/2023-07-nyr/02-data-budget.html#section-5", + "href": "archive/2023-07-nyr/02-data-budget.html#section-5", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n mutate(tip = forcats::fct_rev(tip)) %>% \n ggplot(aes(x = hour, fill = tip)) +\n geom_bar()" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-2", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#logistic-regression-2", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nFind a sigmoid line that separates the two classes" + "objectID": "archive/2023-07-nyr/02-data-budget.html#section-6", + "href": "archive/2023-07-nyr/02-data-budget.html#section-6", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n mutate(tip = forcats::fct_rev(tip)) %>% \n ggplot(aes(x = hour, fill = tip)) +\n geom_bar(position = \"fill\")" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "archive/2023-07-nyr/02-data-budget.html#section-7", + "href": "archive/2023-07-nyr/02-data-budget.html#section-7", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n mutate(tip = forcats::fct_rev(tip)) %>% \n ggplot(aes(x = distance)) +\n geom_histogram(bins = 100) +\n facet_grid(vars(tip))" }, - { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees-1", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees-1", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" + { + "objectID": "archive/2023-07-nyr/02-data-budget.html#section-8", + "href": "archive/2023-07-nyr/02-data-budget.html#section-8", + "title": "2 - Your data budget", + "section": "", + "text": "Stratified sampling would split within response values\n\nBased on our EDA, we know that the source data contains fewer \"no\" tip values than \"yes\". We want to make sure we allot equal proportions of those responses so that both the training and testing data have enough of each to give accurate estimates." }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees-2", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#decision-trees-2", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "archive/2023-07-nyr/02-data-budget.html#stratification", + "href": "archive/2023-07-nyr/02-data-budget.html#stratification", + "title": "2 - Your data budget", + "section": "Stratification", + "text": "Stratification\nUse strata = tip\n\nset.seed(123)\ntaxi_split <- initial_split(taxi, prop = 0.8, strata = tip)\ntaxi_split\n#> <Training/Testing/Total>\n#> <7045/1762/8807>" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "title": "3 - What makes a model?", - "section": "All models are wrong, but some are useful!", - "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" + "objectID": "archive/2023-07-nyr/02-data-budget.html#stratification-1", + "href": "archive/2023-07-nyr/02-data-budget.html#stratification-1", + "title": "2 - Your data budget", + "section": "Stratification", + "text": "Stratification\nStratification often helps, with very little downside" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "title": "3 - What makes a model?", - "section": "Workflows bind preprocessors and models", - "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" + "objectID": "archive/2023-07-nyr/02-data-budget.html#the-whole-game---status-update", + "href": "archive/2023-07-nyr/02-data-budget.html#the-whole-game---status-update", + "title": "2 - Your data budget", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#what-is-wrong-with-this", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#what-is-wrong-with-this", - "title": "3 - What makes a model?", - "section": "What is wrong with this?", - "text": "What is wrong with this?" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#looking-at-predictions", + "href": "archive/2023-07-nyr/04-evaluating-models.html#looking-at-predictions", + "title": "4 - Evaluating models", + "section": "Looking at predictions", + "text": "Looking at predictions\n\naugment(taxi_fit, new_data = taxi_train) %>%\n relocate(tip, .pred_class, .pred_yes, .pred_no)\n#> # A tibble: 7,045 × 10\n#> tip .pred_class .pred_yes .pred_no distance company local dow month hour\n#> <fct> <fct> <dbl> <dbl> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 no no 0.0625 0.937 5.39 Flash … no Sat Mar 12\n#> 2 no yes 0.924 0.0758 18.4 Sun Ta… no Sat Apr 6\n#> 3 no no 0.391 0.609 5.8 other no Tue Jan 10\n#> 4 no no 0.112 0.888 6.85 Flash … no Fri Apr 8\n#> 5 no no 0.129 0.871 9.5 City S… no Wed Jan 7\n#> 6 no no 0.326 0.674 12 other no Fri Apr 11\n#> 7 no no 0.0917 0.908 8.9 Taxi A… no Mon Feb 14\n#> 8 no yes 0.902 0.0980 1.38 other no Fri Apr 16\n#> 9 no no 0.0917 0.908 9.12 Flash … no Wed Apr 9\n#> 10 no yes 0.933 0.0668 2.28 City S… no Thu Apr 16\n#> # ℹ 7,035 more rows" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#why-a-workflow", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#why-a-workflow", - "title": "3 - What makes a model?", - "section": "Why a workflow()? ", - "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering in Advanced tidymodels!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix", + "href": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-1", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-1", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\ntree_spec %>% \n fit(forested ~ ., data = forested_train) \n#> parsnip model object\n#> \n#> n= 5685 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 5685 2550 Yes (0.55145119 0.44854881) \n#> 2) land_type=Tree 3064 300 Yes (0.90208877 0.09791123) *\n#> 3) land_type=Barren,Non-tree vegetation 2621 371 No (0.14154903 0.85845097) \n#> 6) temp_annual_max< 13.395 347 153 Yes (0.55907781 0.44092219) \n#> 12) tree_no_tree=Tree 92 6 Yes (0.93478261 0.06521739) *\n#> 13) tree_no_tree=No tree 255 108 No (0.42352941 0.57647059) *\n#> 7) temp_annual_max>=13.395 2274 177 No (0.07783641 0.92216359) *" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix-1", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(taxi_fit, new_data = taxi_train) %>%\n conf_mat(truth = tip, estimate = .pred_class)\n#> Truth\n#> Prediction yes no\n#> yes 4639 660\n#> no 337 1409" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-2", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-2", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\nworkflow() %>%\n add_formula(forested ~ .) %>%\n add_model(tree_spec) %>%\n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 5685 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 5685 2550 Yes (0.55145119 0.44854881) \n#> 2) land_type=Tree 3064 300 Yes (0.90208877 0.09791123) *\n#> 3) land_type=Barren,Non-tree vegetation 2621 371 No (0.14154903 0.85845097) \n#> 6) temp_annual_max< 13.395 347 153 Yes (0.55907781 0.44092219) \n#> 12) tree_no_tree=Tree 92 6 Yes (0.93478261 0.06521739) *\n#> 13) tree_no_tree=No tree 255 108 No (0.42352941 0.57647059) *\n#> 7) temp_annual_max>=13.395 2274 177 No (0.07783641 0.92216359) *" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix-2", + "href": "archive/2023-07-nyr/04-evaluating-models.html#confusion-matrix-2", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(taxi_fit, new_data = taxi_train) %>%\n conf_mat(truth = tip, estimate = .pred_class) %>%\n autoplot(type = \"heatmap\")" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-3", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#a-model-workflow-3", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\nworkflow(forested ~ ., tree_spec) %>% \n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 5685 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 5685 2550 Yes (0.55145119 0.44854881) \n#> 2) land_type=Tree 3064 300 Yes (0.90208877 0.09791123) *\n#> 3) land_type=Barren,Non-tree vegetation 2621 371 No (0.14154903 0.85845097) \n#> 6) temp_annual_max< 13.395 347 153 Yes (0.55907781 0.44092219) \n#> 12) tree_no_tree=Tree 92 6 Yes (0.93478261 0.06521739) *\n#> 13) tree_no_tree=No tree 255 108 No (0.42352941 0.57647059) *\n#> 7) temp_annual_max>=13.395 2274 177 No (0.07783641 0.92216359) *" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance", + "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n accuracy(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-2", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-2", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code to make a workflow with your own model of choice.\n\nExtension/Challenge: Other than formulas, what kinds of preprocessors are supported?\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-accuracy", + "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-accuracy", + "title": "4 - Evaluating models", + "section": "Dangers of accuracy ", + "text": "Dangers of accuracy \nWe need to be careful of using accuracy() since it can give “good” performance by only predicting one way with imbalanced data\n\naugment(taxi_fit, new_data = taxi_train) %>%\n mutate(.pred_class = factor(\"yes\", levels = c(\"yes\", \"no\"))) %>%\n accuracy(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.706" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#predict-with-your-model", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#predict-with-your-model", - "title": "3 - What makes a model?", - "section": "Predict with your model ", - "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() %>% \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(forested ~ ., tree_spec) %>% \n fit(data = forested_train)" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-1", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n sensitivity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.932" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-3", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-3", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = forested_test)\nWhat do you notice about the structure of the result?\n\n\n\n−+\n03:00" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-2", + "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-2", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n sensitivity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.932\n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n specificity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.681" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-4", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-4", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = forested_test)\nHow does the output compare to the output from predict()?\n\n\n\n−+\n03:00" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-3", + "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-3", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_fit, new_data = taxi_train) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858\n#> 2 specificity binary 0.681\n#> 3 sensitivity binary 0.932" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-4", + "href": "archive/2023-07-nyr/04-evaluating-models.html#metrics-for-model-performance-4", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_fit, new_data = taxi_train) %>%\n group_by(local) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 6 × 4\n#> local .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 yes accuracy binary 0.840\n#> 2 no accuracy binary 0.862\n#> 3 yes specificity binary 0.346\n#> 4 no specificity binary 0.719\n#> 5 yes sensitivity binary 0.969\n#> 6 no sensitivity binary 0.925" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-1", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-1", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\n⚠️ Never predict() with any extracted components!\n\nroundint = FALSE is only to quiet a warning" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#two-class-data", + "href": "archive/2023-07-nyr/04-evaluating-models.html#two-class-data", + "title": "4 - Evaluating models", + "section": "Two class data", + "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-2", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#understand-your-model-2", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#varying-the-threshold", + "href": "archive/2023-07-nyr/04-evaluating-models.html#varying-the-threshold", + "title": "4 - Evaluating models", + "section": "Varying the threshold", + "text": "Varying the threshold" }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-5", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#your-turn-5", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\n\nExtract the model engine object from your fitted workflow and check it out.\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#roc-curves", + "href": "archive/2023-07-nyr/04-evaluating-models.html#roc-curves", + "title": "4 - Evaluating models", + "section": "ROC curves", + "text": "ROC curves\nTo make an ROC (receiver operator characteristic) curve, we:\n\ncalculate the sensitivity and specificity for all possible thresholds\nplot false positive rate (x-axis) versus true positive rate (y-axis)\n\ngiven that sensitivity is the true positive rate, and specificity is the true negative rate. Hence 1 - specificity is the false positive rate.\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." }, { - "objectID": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#the-whole-game---status-update", - "href": "archive/2024-08-posit-conf/intro-03-what-makes-a-model.html#the-whole-game---status-update", - "title": "3 - What makes a model?", - "section": "The whole game - status update", - "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#roc-curves-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#roc-curves-1", + "title": "4 - Evaluating models", + "section": "ROC curves ", + "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_curve(truth = tip, .pred_yes) %>%\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.25 0.486 0.972\n#> 3 0.6 0.705 0.920\n\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_auc(truth = tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.868" }, { - "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#tuning-parameters", - "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#tuning-parameters", - "title": "5 - Tuning models", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#roc-curve-plot", + "href": "archive/2023-07-nyr/04-evaluating-models.html#roc-curve-plot", + "title": "4 - Evaluating models", + "section": "ROC curve plot ", + "text": "ROC curve plot \n\n\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_curve(truth = tip, .pred_yes) %>%\n autoplot()" }, { - "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters", - "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters", - "title": "5 - Tuning models", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn", + "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters-1", - "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#optimize-tuning-parameters-1", - "title": "5 - Tuning models", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-1", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#specifying-tuning-parameters", - "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#specifying-tuning-parameters", - "title": "5 - Tuning models", - "section": "Specifying tuning parameters", - "text": "Specifying tuning parameters\nLet’s take our previous random forest workflow and tag for tuning the minimum number of data points in each node:\n\nrf_spec <- rand_forest(min_n = tune()) %>% \n set_mode(\"classification\")\n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-2", + "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-2", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#try-out-multiple-values", - "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#try-out-multiple-values", - "title": "5 - Tuning models", - "section": "Try out multiple values", - "text": "Try out multiple values\ntune_grid() works similar to fit_resamples() but covers multiple parameter values:\n\nset.seed(22)\nrf_res <- tune_grid(\n rf_wflow,\n forested_folds,\n grid = 5\n)" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-3", + "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-3", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\ntaxi_fit %>%\n augment(taxi_train)\n#> # A tibble: 7,045 × 10\n#> tip distance company local dow month hour .pred_class .pred_yes .pred_no\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int> <fct> <dbl> <dbl>\n#> 1 no 5.39 Flash … no Sat Mar 12 no 0.0625 0.937 \n#> 2 no 18.4 Sun Ta… no Sat Apr 6 yes 0.924 0.0758\n#> 3 no 5.8 other no Tue Jan 10 no 0.391 0.609 \n#> 4 no 6.85 Flash … no Fri Apr 8 no 0.112 0.888 \n#> 5 no 9.5 City S… no Wed Jan 7 no 0.129 0.871 \n#> 6 no 12 other no Fri Apr 11 no 0.326 0.674 \n#> 7 no 8.9 Taxi A… no Mon Feb 14 no 0.0917 0.908 \n#> 8 no 1.38 other no Fri Apr 16 yes 0.902 0.0980\n#> 9 no 9.12 Flash … no Wed Apr 9 no 0.0917 0.908 \n#> 10 no 2.28 City S… no Thu Apr 16 yes 0.933 0.0668\n#> # ℹ 7,035 more rows\n\nWe call this “resubstitution” or “repredicting the training set”" }, { - "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#compare-results", - "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#compare-results", - "title": "5 - Tuning models", - "section": "Compare results", - "text": "Compare results\nInspecting results and selecting the best-performing hyperparameter(s):\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 21 roc_auc binary 0.972 10 0.00295 Preprocessor1_Model4\n#> 2 6 roc_auc binary 0.972 10 0.00303 Preprocessor1_Model2\n#> 3 31 roc_auc binary 0.972 10 0.00317 Preprocessor1_Model3\n#> 4 13 roc_auc binary 0.972 10 0.00311 Preprocessor1_Model5\n#> 5 33 roc_auc binary 0.972 10 0.00322 Preprocessor1_Model1\n\nbest_parameter <- select_best(rf_res)\nbest_parameter\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 21 Preprocessor1_Model4\n\ncollect_metrics() and autoplot() are also available." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-4", + "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-4", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858\n\nWe call this a “resubstitution estimate”" }, { - "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#the-final-fit", - "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#the-final-fit", - "title": "5 - Tuning models", - "section": "The final fit", - "text": "The final fit\n\nrf_wflow <- finalize_workflow(rf_wflow, best_parameter)\n\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.906 Preprocessor1_Model1\n#> 2 roc_auc binary 0.970 Preprocessor1_Model1\n#> 3 brier_class binary 0.0656 Preprocessor1_Model1" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-5", + "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-5", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858" }, { - "objectID": "archive/2024-08-posit-conf/intro-05-tuning-models.html#your-turn", - "href": "archive/2024-08-posit-conf/intro-05-tuning-models.html#your-turn", - "title": "5 - Tuning models", - "section": "Your turn", - "text": "Your turn\n\nModify your model workflow to tune one or more parameters.\nUse grid search to find the best parameter(s).\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-6", + "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-6", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.858\n\n\n\ntaxi_fit %>%\n augment(taxi_test) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.795\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#looking-at-the-predictors", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#looking-at-the-predictors", - "title": "Extras - Recipes", - "section": "Looking at the predictors", - "text": "Looking at the predictors\n\nforested_train\n#> # A tibble: 5,685 × 19\n#> forested year elevation eastness northness roughness tree_no_tree dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon\n#> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 No 2016 464 -5 -99 7 No tree 1.71 282 9.76 -4.44 16.6 2.96 191 1534 4 -121.\n#> 2 Yes 2016 166 92 37 7 Tree 6 1298 10.2 0.72 14.3 6.12 60 747 33 -122.\n#> 3 No 2016 644 -85 -52 24 No tree 0.67 288 8.77 -6.32 14.6 2.98 219 1396 0 -120.\n#> 4 Yes 2014 1285 4 99 79 Tree 1.91 1621 5.61 -2.48 9.48 1.73 88 545 74 -123.\n#> 5 Yes 2013 822 87 48 68 Tree 1.95 2200 8.62 -0.68 12.9 4.35 147 861 48 -121.\n#> 6 Yes 2017 3 6 -99 5 Tree 7.93 2211 10.6 3.77 14.2 7.02 34 578 79 -124.\n#> 7 Yes 2014 2041 -95 28 49 Tree -4.22 1551 0.75 -9.47 5.17 -3.66 73 481 48 -120.\n#> 8 Yes 2015 1009 -8 99 72 Tree 1.72 2396 6.59 -2.98 11.3 1.88 92 781 76 -122.\n#> 9 No 2017 436 -98 19 10 No tree 1.8 234 9.8 -4.23 16.3 3.32 178 1527 0 -119.\n#> 10 No 2018 775 63 76 103 No tree 0.62 432 8.51 -5.5 13.7 3.32 241 1237 7 -120.\n#> # ℹ 5,675 more rows\n#> # ℹ 2 more variables: lat <dbl>, land_type <fct>" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-1", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse augment() and and a metric function to compute a classification metric like brier_class().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#working-with-other-models", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#working-with-other-models", - "title": "Extras - Recipes", - "section": "Working with other models", - "text": "Working with other models\nSome models can’t handle non-numeric data\n\nLinear Regression\nK Nearest Neighbors\n\n\n\nSome models struggle if numeric predictors aren’t scaled\n\nK Nearest Neighbors\nAnything using gradient descent" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-7", + "href": "archive/2023-07-nyr/04-evaluating-models.html#dangers-of-overfitting-7", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n brier_class(tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.113\n\n\n\ntaxi_fit %>%\n augment(taxi_test) %>%\n brier_class(tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.152\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#types-of-needed-preprocessing", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#types-of-needed-preprocessing", - "title": "Extras - Recipes", - "section": "Types of needed preprocessing", - "text": "Types of needed preprocessing\n\nDo qualitative predictors require a numeric encoding?\nShould columns with a single unique value be removed?\nDoes the model struggle with missing data?\nDoes the model struggle with correlated predictors?\nShould predictors be centered and scaled?\nIs it helpful to transform predictors to be more symmetric?\n\n\nhttps://www.tmwr.org/pre-proc-table.html" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation", + "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing", - "title": "Extras - Recipes", - "section": "Two types of preprocessing", - "text": "Two types of preprocessing" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-1", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing-1", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-preprocessing-1", - "title": "Extras - Recipes", - "section": "Two types of preprocessing", - "text": "Two types of preprocessing" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-2", + "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-2", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#general-definitions", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#general-definitions", - "title": "Extras - Recipes", - "section": "General definitions", - "text": "General definitions\n\nData preprocessing is what you do to make your model successful.\nFeature engineering is what you do to the original predictors to make the model do the least work to perform great." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-2", + "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-2", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(taxi_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6340/705]> Fold01\n#> 2 <split [6340/705]> Fold02\n#> 3 <split [6340/705]> Fold03\n#> 4 <split [6340/705]> Fold04\n#> 5 <split [6340/705]> Fold05\n#> 6 <split [6341/704]> Fold06\n#> 7 <split [6341/704]> Fold07\n#> 8 <split [6341/704]> Fold08\n#> 9 <split [6341/704]> Fold09\n#> 10 <split [6341/704]> Fold10" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#working-with-dates", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#working-with-dates", - "title": "Extras - Recipes", - "section": "Working with dates", - "text": "Working with dates\nDatetime variables are automatically converted to an integer if given as a raw predictor. To avoid this, it can be re-encoded as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nLeap year\nIndicators for holidays" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-3", + "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-3", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWhat is in this?\n\ntaxi_folds <- vfold_cv(taxi_train)\ntaxi_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <6340/705/7045>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <6340/705/7045>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <6340/705/7045>\n\n\nTalk about a list column, storing non-atomic types in dataframe" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-transformations", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#two-types-of-transformations", - "title": "Extras - Recipes", - "section": "Two types of transformations", - "text": "Two types of transformations\n\n\n\nStatic\n\nSquare root, log, inverse\nDummies for known levels\nDate time extractions\n\n\nTrained\n\nCentering & scaling\nImputation\nPCA\nAnything for unknown factor levels\n\n\n\nTrained methods need to calculate sufficient information to be applied again." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-4", + "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-4", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(taxi_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [5636/1409]> Fold1\n#> 2 <split [5636/1409]> Fold2\n#> 3 <split [5636/1409]> Fold3\n#> 4 <split [5636/1409]> Fold4\n#> 5 <split [5636/1409]> Fold5" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#the-recipes-package", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#the-recipes-package", - "title": "Extras - Recipes", - "section": "The recipes package", - "text": "The recipes package\n\n\nModular + extensible\nWorks well with pipes ,|> and %>%\nDeferred evaluation\nIsolates test data from training data\nCan do things formulas can’t" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-5", + "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-5", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(taxi_train, strata = tip)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6340/705]> Fold01\n#> 2 <split [6340/705]> Fold02\n#> 3 <split [6340/705]> Fold03\n#> 4 <split [6340/705]> Fold04\n#> 5 <split [6340/705]> Fold05\n#> 6 <split [6340/705]> Fold06\n#> 7 <split [6341/704]> Fold07\n#> 8 <split [6341/704]> Fold08\n#> 9 <split [6341/704]> Fold09\n#> 10 <split [6342/703]> Fold10\n\n\nStratification often helps, with very little downside" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-6", + "href": "archive/2023-07-nyr/04-evaluating-models.html#cross-validation-6", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\ntaxi_folds <- vfold_cv(taxi_train, v = 10, strata = tip)\ntaxi_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6340/705]> Fold01\n#> 2 <split [6340/705]> Fold02\n#> 3 <split [6340/705]> Fold03\n#> 4 <split [6340/705]> Fold04\n#> 5 <split [6340/705]> Fold05\n#> 6 <split [6340/705]> Fold06\n#> 7 <split [6341/704]> Fold07\n#> 8 <split [6341/704]> Fold08\n#> 9 <split [6341/704]> Fold09\n#> 10 <split [6342/703]> Fold10\n\n\nSet the seed when creating resamples" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-1", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-1", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())\n\n\nStart by calling recipe() to denote the data source and variables used." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#fit-our-model-to-the-resamples", + "href": "archive/2023-07-nyr/04-evaluating-models.html#fit-our-model-to-the-resamples", + "title": "4 - Evaluating models", + "section": "Fit our model to the resamples", + "text": "Fit our model to the resamples\n\ntaxi_res <- fit_resamples(taxi_wflow, taxi_folds)\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [6340/705]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 2 <split [6340/705]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 3 <split [6340/705]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 4 <split [6340/705]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 5 <split [6340/705]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 6 <split [6340/705]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 7 <split [6341/704]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 8 <split [6341/704]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 9 <split [6341/704]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 10 <split [6342/703]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]>" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-2", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-2", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%\n  step_normalize(all_numeric_predictors())\n\n\nSpecify what actions to take by adding step_*()s." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance", + "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\ntaxi_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.793 10 0.00293 Preprocessor1_Model1\n#> 2 roc_auc binary 0.809 10 0.00461 Preprocessor1_Model1\n\n\nWe can reliably measure performance using only the training data 🎉" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-3", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#how-to-write-a-recipe-3", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%   step_normalize(all_numeric_predictors())\n\n\nUse {tidyselect} and recipes-specific selectors to denote affected variables." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#comparing-metrics", + "href": "archive/2023-07-nyr/04-evaluating-models.html#comparing-metrics", + "title": "4 - Evaluating models", + "section": "Comparing metrics ", + "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\ntaxi_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.793 10\n#> 2 roc_auc 0.809 10\n\n\nThe ROC AUC previously was\n\n0.87 for the training set\n0.81 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe", - "title": "Extras - Recipes", - "section": "Using a recipe", - "text": "Using a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) %>%\n  step_dummy(all_nominal_predictors()) %>%\n  step_zv(all_predictors()) %>%\n  step_log(canopy_cover, offset = 0.5) %>%   step_normalize(all_numeric_predictors())\n\n\nSave the recipe we like so that we can use it in various places, e.g., with different models." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-1", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_taxi <- control_resamples(save_pred = TRUE)\ntaxi_res <- fit_resamples(taxi_wflow, taxi_folds, control = ctrl_taxi)\n\ntaxi_preds <- collect_predictions(taxi_res)\ntaxi_preds\n#> # A tibble: 7,045 × 7\n#> id .pred_yes .pred_no .row .pred_class tip .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 Fold01 0.936 0.0638 10 yes no Preprocessor1_Model1\n#> 2 Fold01 0.898 0.102 20 yes no Preprocessor1_Model1\n#> 3 Fold01 0.898 0.102 47 yes no Preprocessor1_Model1\n#> 4 Fold01 0.101 0.899 51 no no Preprocessor1_Model1\n#> 5 Fold01 0.871 0.129 59 yes no Preprocessor1_Model1\n#> 6 Fold01 0.0815 0.918 60 no no Preprocessor1_Model1\n#> 7 Fold01 0.162 0.838 92 no no Preprocessor1_Model1\n#> 8 Fold01 0.26 0.74 97 no no Preprocessor1_Model1\n#> 9 Fold01 0.274 0.726 98 no no Preprocessor1_Model1\n#> 10 Fold01 0.804 0.196 104 yes no Preprocessor1_Model1\n#> # ℹ 7,035 more rows" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe-with-workflows", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe-with-workflows", - "title": "Extras - Recipes", - "section": "Using a recipe with workflows", - "text": "Using a recipe with workflows\nRecipes are typically combined with a model in a workflow() object:\n\n\nforested_wflow <- workflow() %>%\n  add_recipe(forested_rec) %>%\n  add_model(linear_reg())" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-2", + "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-2", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\ntaxi_preds %>% \n group_by(id) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.793\n#> 2 Fold02 accuracy binary 0.8 \n#> 3 Fold03 accuracy binary 0.786\n#> 4 Fold04 accuracy binary 0.804\n#> 5 Fold05 accuracy binary 0.796\n#> 6 Fold06 accuracy binary 0.789\n#> 7 Fold07 accuracy binary 0.793\n#> 8 Fold08 accuracy binary 0.808\n#> 9 Fold09 accuracy binary 0.783\n#> 10 Fold10 accuracy binary 0.780\n#> # ℹ 20 more rows" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#recipes-are-estimated", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#recipes-are-estimated", - "title": "Extras - Recipes", - "section": "Recipes are estimated", - "text": "Recipes are estimated\nEvery preprocessing step in a recipe that involved calculations uses the training set. For example:\n\nLevels of a factor\nDetermination of zero-variance\nNormalization\nFeature extraction\n\nOnce a recipe is added to a workflow, this occurs when fit() is called." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#where-are-the-fitted-models", + "href": "archive/2023-07-nyr/04-evaluating-models.html#where-are-the-fitted-models", + "title": "4 - Evaluating models", + "section": "Where are the fitted models? ", + "text": "Where are the fitted models? \n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [6340/705]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [6340/705]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [6340/705]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [6340/705]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [6340/705]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [6340/705]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [6341/704]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [6341/704]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [6341/704]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [6342/703]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#debugging-a-recipe", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#debugging-a-recipe", - "title": "Extras - Recipes", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. forested_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4.\n\n\n\n\nAnother function, bake(), is analogous to predict(), and gives you the processed data back." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#bootstrapping", + "href": "archive/2023-07-nyr/04-evaluating-models.html#bootstrapping", + "title": "4 - Evaluating models", + "section": "Bootstrapping", + "text": "Bootstrapping" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#your-turn", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#your-turn", - "title": "Extras - Recipes", - "section": "Your turn", - "text": "Your turn\n\n\nTake the recipe and prep() then bake() it to see what the resulting data set looks like.\nTry removing steps to see how the result changes.\n\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#bootstrapping-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#bootstrapping-1", + "title": "4 - Evaluating models", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(taxi_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7045/2561]> Bootstrap01\n#> 2 <split [7045/2577]> Bootstrap02\n#> 3 <split [7045/2648]> Bootstrap03\n#> 4 <split [7045/2616]> Bootstrap04\n#> 5 <split [7045/2616]> Bootstrap05\n#> 6 <split [7045/2599]> Bootstrap06\n#> 7 <split [7045/2654]> Bootstrap07\n#> 8 <split [7045/2593]> Bootstrap08\n#> 9 <split [7045/2624]> Bootstrap09\n#> 10 <split [7045/2615]> Bootstrap10\n#> # ℹ 15 more rows" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#printing-a-recipe", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#printing-a-recipe", - "title": "Extras - Recipes", - "section": "Printing a recipe", - "text": "Printing a recipe\n\nforested_rec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Operations\n#> • Dummy variables from: all_nominal_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Log transformation on: canopy_cover\n#> • Centering and scaling for: all_numeric_predictors()" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game---status-update", + "href": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game---status-update", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#prepping-a-recipe", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#prepping-a-recipe", - "title": "Extras - Recipes", - "section": "Prepping a recipe", - "text": "Prepping a recipe\n\nprep(forested_rec)\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Training information\n#> Training data contained 5685 data points and no incomplete rows.\n#> \n#> ── Operations\n#> • Dummy variables from: tree_no_tree and land_type | Trained\n#> • Zero variance filter removed: <none> | Trained\n#> • Log transformation on: canopy_cover | Trained\n#> • Centering and scaling for: year and elevation, ... | Trained" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-3", + "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-3", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the function)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#baking-a-recipe", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#baking-a-recipe", - "title": "Extras - Recipes", - "section": "Baking a recipe", - "text": "Baking a recipe\n\nprep(forested_rec) %>%\n bake(new_data = forested_train)\n#> # A tibble: 5,685 × 20\n#> year elevation eastness northness roughness dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat forested\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> \n#> 1 0.206 -0.450 -0.0203 -1.38 -0.874 -0.169 -0.864 0.532 -0.403 0.959 -0.0702 0.755 1.21 -0.566 -0.175 -0.988 No \n#> 2 0.206 -1.07 1.38 0.563 -0.874 1.33 0.132 0.726 1.24 0.159 1.34 -1.00 -1.00 0.494 -0.856 1.10 Yes \n#> 3 0.206 -0.0762 -1.17 -0.711 -0.506 -0.533 -0.858 0.116 -1.00 0.248 -0.0613 1.13 0.820 -1.73 0.270 0.175 No \n#> 4 -0.413 1.26 0.109 1.45 0.683 -0.0987 0.448 -1.21 0.221 -1.56 -0.621 -0.625 -1.57 0.917 -1.52 0.654 Yes \n#> 5 -0.723 0.294 1.31 0.721 0.445 -0.0847 1.02 0.0529 0.795 -0.346 0.552 0.166 -0.683 0.690 -0.419 1.32 Yes \n#> 6 0.516 -1.41 0.138 -1.38 -0.917 2.01 1.03 0.894 2.21 0.127 1.75 -1.35 -1.48 0.951 -1.84 -0.338 Yes \n#> 7 -0.413 2.83 -1.32 0.435 0.0343 -2.25 0.380 -3.26 -2.01 -3.09 -3.03 -0.826 -1.75 0.690 -0.0159 1.65 Yes \n#> 8 -0.103 0.682 -0.0636 1.45 0.532 -0.165 1.21 -0.801 0.0620 -0.915 -0.554 -0.571 -0.908 0.931 -0.636 -1.33 Yes \n#> 9 0.516 -0.508 -1.36 0.306 -0.809 -0.137 -0.911 0.549 -0.336 0.856 0.0909 0.581 1.19 -1.73 0.760 -0.209 No \n#> 10 0.826 0.196 0.960 1.12 1.20 -0.550 -0.717 0.00667 -0.741 -0.0616 0.0909 1.43 0.373 -0.296 0.155 -0.204 No \n#> # ℹ 5,675 more rows\n#> # ℹ 3 more variables: tree_no_tree_No.tree <dbl>, land_type_Non.tree.vegetation <dbl>, land_type_Tree <dbl>" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#monte-carlo-cross-validation", + "href": "archive/2023-07-nyr/04-evaluating-models.html#monte-carlo-cross-validation", + "title": "4 - Evaluating models", + "section": "Monte Carlo Cross-Validation ", + "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(taxi_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5283/1762]> Resample01\n#> 2 <split [5283/1762]> Resample02\n#> 3 <split [5283/1762]> Resample03\n#> 4 <split [5283/1762]> Resample04\n#> 5 <split [5283/1762]> Resample05\n#> 6 <split [5283/1762]> Resample06\n#> 7 <split [5283/1762]> Resample07\n#> 8 <split [5283/1762]> Resample08\n#> 9 <split [5283/1762]> Resample09\n#> 10 <split [5283/1762]> Resample10" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\nOnce a recipe as been estimated, there are various bits of information saved in it.\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#validation-set", + "href": "archive/2023-07-nyr/04-evaluating-models.html#validation-set", + "title": "4 - Evaluating models", + "section": "Validation set ", + "text": "Validation set \n\nset.seed(853)\nvalidation_split(taxi_train, strata = tip)\n#> # Validation Set Split (0.75/0.25) using stratification \n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5283/1762]> validation\n\n\nA validation set is just another type of resample" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#your-turn-1", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#your-turn-1", - "title": "Extras - Recipes", - "section": "Your turn", - "text": "Your turn\n\nTake a prepped recipe and use the tidy() function on it.\nUse the number argument to inspect different steps.\n\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#random-forest-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#random-forest-1", + "title": "4 - Evaluating models", + "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this tomorrow!), as long as there are enough trees" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe-1", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe-1", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\n\nprep(forested_rec) %>%\n tidy()\n#> # A tibble: 4 × 6\n#> number operation type trained skip id \n#> <int> <chr> <chr> <lgl> <lgl> <chr> \n#> 1 1 step dummy TRUE FALSE dummy_jlmcG \n#> 2 2 step zv TRUE FALSE zv_mYCvS \n#> 3 3 step log TRUE FALSE log_eme6b \n#> 4 4 step normalize TRUE FALSE normalize_ScVef" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#create-a-random-forest-model", + "href": "archive/2023-07-nyr/04-evaluating-models.html#create-a-random-forest-model", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe-2", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#tidying-a-recipe-2", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\n\nprep(forested_rec) %>%\n tidy(number = 1)\n#> # A tibble: 3 × 3\n#> terms columns id \n#> <chr> <chr> <chr> \n#> 1 tree_no_tree No tree dummy_jlmcG\n#> 2 land_type Non-tree vegetation dummy_jlmcG\n#> 3 land_type Tree dummy_jlmcG" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#create-a-random-forest-model-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#create-a-random-forest-model-1", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_wflow <- workflow(tip ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe-in-tidymodels", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#using-a-recipe-in-tidymodels", - "title": "Extras - Recipes", - "section": "Using a recipe in tidymodels", - "text": "Using a recipe in tidymodels\nThe recommended way to use a recipe in tidymodels is to use it as part of a workflow().\n\nforested_wflow <- workflow() %>% \n add_recipe(forested_rec) %>% \n add_model(linear_reg())\n\nWhen used in this way, you don’t need to worry about prep() and bake() as it is handled for you." + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-4", + "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-4", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" }, { - "objectID": "archive/2024-08-posit-conf/intro-extra-recipes.html#more-information", - "href": "archive/2024-08-posit-conf/intro-extra-recipes.html#more-information", - "title": "Extras - Recipes", - "section": "More information", - "text": "More information\n\nhttps://recipes.tidymodels.org/\nhttps://www.tmwr.org/recipes.html" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-3", + "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluating-model-performance-3", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nctrl_taxi <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, taxi_folds, control = ctrl_taxi)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.813 10 0.00305 Preprocessor1_Model1\n#> 2 roc_auc binary 0.832 10 0.00513 Preprocessor1_Model1" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", - "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", - "title": "Extras - workflowsets", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", + "href": "archive/2023-07-nyr/04-evaluating-models.html#how-can-we-compare-multiple-model-workflows-at-once", + "title": "4 - Evaluating models", "section": "How can we compare multiple model workflows at once?", "text": "How can we compare multiple model workflows at once?" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set", - "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set", - "title": "Extras - workflowsets", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set", + "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set", + "title": "4 - Evaluating models", "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nwf_set <- workflow_set(list(forested ~ .), list(tree_spec, rf_spec))\nwf_set\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" + "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec))\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", - "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", - "title": "Extras - workflowsets", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set-1", + "title": "4 - Evaluating models", "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nwf_set_fit <- wf_set |>\n workflow_map(\"fit_resamples\", resamples = forested_folds)\nwf_set_fit\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" + "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = taxi_folds)\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", - "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", - "title": "Extras - workflowsets", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set-2", + "href": "archive/2023-07-nyr/04-evaluating-models.html#evaluate-a-workflow-set-2", + "title": "4 - Evaluating models", "section": "Evaluate a workflow set", - "text": "Evaluate a workflow set\n\nwf_set_fit |>\n rank_results()\n#> # A tibble: 6 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_for… Prepro… accura… 0.753 0.00435 10 formula rand… 1\n#> 2 formula_rand_for… Prepro… brier_… 0.167 0.00316 10 formula rand… 1\n#> 3 formula_rand_for… Prepro… roc_auc 0.757 0.00992 10 formula rand… 1\n#> 4 formula_decision… Prepro… accura… 0.704 0.00653 10 formula deci… 2\n#> 5 formula_decision… Prepro… brier_… 0.214 0.00349 10 formula deci… 2\n#> 6 formula_decision… Prepro… roc_auc 0.692 0.00496 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" + "text": "Evaluate a workflow set\n\nworkflow_set(list(tip ~ .), list(tree_spec, rf_spec)) %>%\n workflow_map(\"fit_resamples\", resamples = taxi_folds) %>%\n rank_results()\n#> # A tibble: 4 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_for… Prepro… accura… 0.813 0.00339 10 formula rand… 1\n#> 2 formula_rand_for… Prepro… roc_auc 0.833 0.00528 10 formula rand… 1\n#> 3 formula_decision… Prepro… accura… 0.793 0.00293 10 formula deci… 2\n#> 4 formula_decision… Prepro… roc_auc 0.809 0.00461 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#your-turn", - "href": "archive/2025-08-nyr/slides/intro-extra-workflowsets.html#your-turn", - "title": "Extras - workflowsets", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-5", + "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-5", + "title": "4 - Evaluating models", "section": "Your turn", - "text": "Your turn\n\nWhen do you think a workflow set would be useful?\nDiscuss with your neighbors!\n\n\n\n−+\n03:00" + "text": "Your turn\n\nWhen do you think a workflow set would be useful?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-06-wrapping-up.html#your-turn", - "href": "archive/2025-08-nyr/slides/intro-06-wrapping-up.html#your-turn", - "title": "6 - Wrapping up", - "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game---status-update-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game---status-update-1", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "archive/2025-08-nyr/slides/intro-06-wrapping-up.html#resources-to-keep-learning", - "href": "archive/2025-08-nyr/slides/intro-06-wrapping-up.html#resources-to-keep-learning", - "title": "6 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://tidymodels.aml4td.org/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#the-final-fit", + "href": "archive/2023-07-nyr/04-evaluating-models.html#the-final-fit", + "title": "4 - Evaluating models", + "section": "The final fit ", + "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# taxi_split has train + test info\nfinal_fit <- last_fit(rf_wflow, taxi_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [7045/1762]> train/test split <tibble> <tibble> <tibble> <workflow>" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#looking-at-predictions", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#looking-at-predictions", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit", + "href": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit", "title": "4 - Evaluating models", - "section": "Looking at predictions", - "text": "Looking at predictions\n\naugment(forested_fit, new_data = forested_train)\n#> # A tibble: 8,749 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness roughness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 0.931 0.0690 Yes 1997 66 82 10\n#> 2 Yes 0.983 0.0172 No 1997 284 -99 58\n#> 3 Yes 0.960 0.0401 Yes 2022 130 86 15\n#> 4 Yes 0.870 0.130 Yes 2021 202 -55 3\n#> 5 Yes 0.823 0.177 Yes 1995 75 -89 1\n#> 6 Yes 0.758 0.242 No 1995 110 -53 5\n#> 7 Yes 0.823 0.177 Yes 2022 111 73 12\n#> 8 No 0.467 0.533 Yes 1997 230 96 14\n#> 9 Yes 0.983 0.0172 Yes 2002 160 -88 13\n#> 10 Yes 0.871 0.129 Yes 2020 39 9 6\n#> # ℹ 8,739 more rows\n#> # ℹ 14 more variables: tree_no_tree <fct>, dew_temp <dbl>, precip_annual <dbl>,\n#> # temp_annual_mean <dbl>, temp_annual_min <dbl>, temp_annual_max <dbl>,\n#> # temp_january_min <dbl>, vapor_min <dbl>, vapor_max <dbl>,\n#> # canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>, county <fct>" + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.810 Preprocessor1_Model1\n#> 2 roc_auc binary 0.817 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit-1", + "href": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit-1", "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix" + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 1,762 × 7\n#> id .pred_yes .pred_no .row .pred_class tip .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 train/test split 0.732 0.268 10 yes no Preprocessor1_Mo…\n#> 2 train/test split 0.827 0.173 29 yes yes Preprocessor1_Mo…\n#> 3 train/test split 0.899 0.101 35 yes yes Preprocessor1_Mo…\n#> 4 train/test split 0.914 0.0856 42 yes yes Preprocessor1_Mo…\n#> 5 train/test split 0.911 0.0889 47 yes no Preprocessor1_Mo…\n#> 6 train/test split 0.848 0.152 54 yes yes Preprocessor1_Mo…\n#> 7 train/test split 0.580 0.420 59 yes yes Preprocessor1_Mo…\n#> 8 train/test split 0.912 0.0876 62 yes yes Preprocessor1_Mo…\n#> 9 train/test split 0.810 0.190 63 yes yes Preprocessor1_Mo…\n#> 10 train/test split 0.960 0.0402 69 yes yes Preprocessor1_Mo…\n#> # ℹ 1,752 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix-1", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit-2", + "href": "archive/2023-07-nyr/04-evaluating-models.html#what-is-in-final_fit-2", "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) |>\n conf_mat(truth = forested, estimate = .pred_class)\n#> Truth\n#> Prediction Yes No\n#> Yes 5884 869\n#> No 438 1558" + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 7045 \n#> Number of independent variables: 6 \n#> Mtry: 2 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.1373147\n\n\nUse this for prediction on new data, like for deploying" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix-2", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#confusion-matrix-2", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game", + "href": "archive/2023-07-nyr/04-evaluating-models.html#the-whole-game", "title": "4 - Evaluating models", - "section": "Confusion matrix ", - "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) |>\n conf_mat(truth = forested, estimate = .pred_class) |>\n autoplot(type = \"heatmap\")" + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance", + "objectID": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-6", + "href": "archive/2023-07-nyr/04-evaluating-models.html#your-turn-6", "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851" + "section": "Your turn", + "text": "Your turn\n\nEnd of the day discussion!\nWhich model do you think you would decide to use?\nWhat surprised you the most?\nWhat is one thing you are looking forward to for tomorrow?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-accuracy", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-accuracy", - "title": "4 - Evaluating models", - "section": "Dangers of accuracy ", - "text": "Dangers of accuracy \nWe need to be careful of using accuracy() since it can give “good” performance by only predicting one way with imbalanced data:\n\naugment(forested_fit, new_data = forested_train) %>%\n mutate(.pred_class = factor(\"Yes\", levels = c(\"Yes\", \"No\"))) %>%\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.723" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#working-with-our-predictors", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#working-with-our-predictors", + "title": "1 - Feature Engineering", + "section": "Working with our predictors", + "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\nThe model requires them in a different format (e.g. dummy variables for linear regression).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-1", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.931" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#what-is-feature-engineering", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#what-is-feature-engineering", + "title": "1 - Feature Engineering", + "section": "What is feature engineering?", + "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPrincipal component analysis (PCA) feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection (FES)." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-2", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-2", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) |>\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.931\n\n\n\naugment(forested_fit, new_data = forested_train) |>\n specificity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.642" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#example-dates", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#example-dates", + "title": "1 - Feature Engineering", + "section": "Example: Dates", + "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-3", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-3", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\nforested_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(forested_fit, new_data = forested_train) |>\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n#> 2 specificity binary 0.642\n#> 3 sensitivity binary 0.931" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#general-definitions", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#general-definitions", + "title": "1 - Feature Engineering", + "section": "General definitions ", + "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-4", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#metrics-for-model-performance-4", - "title": "4 - Evaluating models", - "section": "Metrics for model performance ", - "text": "Metrics for model performance \nMetrics and metric sets work with grouped data frames!\n\naugment(forested_fit, new_data = forested_train) |>\n group_by(tree_no_tree) |>\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 2 × 4\n#> tree_no_tree .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 Tree accuracy binary 0.875\n#> 2 No tree accuracy binary 0.807" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#hotel-data", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#hotel-data", + "title": "1 - Feature Engineering", + "section": "Hotel Data ", + "text": "Hotel Data \nWe’ll use data on hotels to predict the cost of a room.\nThe data are in the modeldatatoo package. We’ll sample down the data and refactor some columns:\n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn", - "title": "4 - Evaluating models", + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#data-splitting-strategy", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#data-splitting-strategy", + "title": "1 - Feature Engineering", + "section": "Data splitting strategy", + "text": "Data splitting strategy" + }, + { + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#data-spending", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#data-spending", + "title": "1 - Feature Engineering", + "section": "Data Spending ", + "text": "Data Spending \nLet’s split the data into a training set (75%) and testing set (25%):\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)" + }, + { + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn", + "title": "1 - Feature Engineering", "section": "Your turn", - "text": "Your turn\n\nApply the forested_metrics metric set to augment()\noutput grouped by tree_no_tree.\nDo any metrics differ substantially between groups?\n\n\n\n−+\n05:00\n\n\n\n\nThe specificity for \"Tree\" is a good bit lower than it is for \"No tree\".\nSpecificity is the proportion of negatives that are correctly identified as negatives. “Negative” is the non-event level of the outcome, i.e. “non-forested.” So, when this index classifies the plot as having a tree, the model does not do well at correctly identifying the plot as non-forested when it is indeed non-forested." + "text": "Your turn\nLet’s take some time and investigate the training data. The outcome is avg_price_per_room.\nAre there any interesting characteristics of the data?\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#two-class-data", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#two-class-data", - "title": "4 - Evaluating models", - "section": "Two class data", - "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#resampling-strategy", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#resampling-strategy", + "title": "1 - Feature Engineering", + "section": "Resampling Strategy", + "text": "Resampling Strategy" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#varying-the-threshold", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#varying-the-threshold", - "title": "4 - Evaluating models", - "section": "Varying the threshold", - "text": "Varying the threshold" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#resampling-strategy-1", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#resampling-strategy-1", + "title": "1 - Feature Engineering", + "section": "Resampling Strategy ", + "text": "Resampling Strategy \nWe’ll use simple 10-fold cross-validation (stratified sampling):\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)\nhotel_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [3372/377]> Fold01\n#> 2 <split [3373/376]> Fold02\n#> 3 <split [3373/376]> Fold03\n#> 4 <split [3373/376]> Fold04\n#> 5 <split [3373/376]> Fold05\n#> 6 <split [3374/375]> Fold06\n#> 7 <split [3375/374]> Fold07\n#> 8 <split [3376/373]> Fold08\n#> 9 <split [3376/373]> Fold09\n#> 10 <split [3376/373]> Fold10" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves", - "title": "4 - Evaluating models", - "section": "ROC curves", - "text": "ROC curves\n\n\nFor an ROC (receiver operator characteristic) curve, we plot\n\nthe false positive rate (1 - specificity) on the x-axis\nthe true positive rate (sensitivity) on the y-axis\n\nwith sensitivity and specificity calculated at all possible thresholds.\n\n\n\n\n\n\n\n\n\n\n\n\nROC curves are insensitive to class imbalance." + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#prepare-your-data-for-modeling", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#prepare-your-data-for-modeling", + "title": "1 - Feature Engineering", + "section": "Prepare your data for modeling ", + "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of preprocessing and feature engineering steps.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves-1", - "title": "4 - Evaluating models", - "section": "ROC curves", - "text": "ROC curves\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#a-first-recipe", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#a-first-recipe", + "title": "1 - Feature Engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves-2", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curves-2", - "title": "4 - Evaluating models", - "section": "ROC curves ", - "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(forested_fit, new_data = forested_train) |> \n roc_curve(truth = forested, .pred_Yes) |>\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.143 0.267 0.990\n#> 3 0.385 0.571 0.951\n\naugment(forested_fit, new_data = forested_train) |> \n roc_auc(truth = forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.881" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#a-first-recipe-1", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#a-first-recipe-1", + "title": "1 - Feature Engineering", + "section": "A first recipe ", + "text": "A first recipe \n\nsummary(hotel_rec)\n#> # A tibble: 28 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 lead_time <chr [2]> predictor original\n#> 2 arrival_date_day_of_month <chr [2]> predictor original\n#> 3 stays_in_weekend_nights <chr [2]> predictor original\n#> 4 stays_in_week_nights <chr [2]> predictor original\n#> 5 adults <chr [2]> predictor original\n#> 6 children <chr [2]> predictor original\n#> 7 babies <chr [2]> predictor original\n#> 8 meal <chr [3]> predictor original\n#> 9 country <chr [3]> predictor original\n#> 10 market_segment <chr [3]> predictor original\n#> # ℹ 18 more rows\n\nThe type column contains information on the variables" + }, + { + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-1", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-1", + "title": "1 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\nWhat do you think are in the type vectors for the lead_time and country columns?\n\n\n\n−+\n02:00" + }, + { + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#create-indicator-variables", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#create-indicator-variables", + "title": "1 - Feature Engineering", + "section": "Create indicator variables ", + "text": "Create indicator variables \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns.\nstep_dummy() records the levels of the categorical predictors in the training set." + }, + { + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#filter-out-constant-columns", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#filter-out-constant-columns", + "title": "1 - Feature Engineering", + "section": "Filter out constant columns ", + "text": "Filter out constant columns \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" + }, + { + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#normalization", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#normalization", + "title": "1 - Feature Engineering", + "section": "Normalization ", + "text": "Normalization \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curve-plot", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#roc-curve-plot", - "title": "4 - Evaluating models", - "section": "ROC curve plot ", - "text": "ROC curve plot \n\n\naugment(forested_fit, \n new_data = forested_train) |> \n roc_curve(truth = forested, \n .pred_Yes) |>\n autoplot()" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#reduce-correlation", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#reduce-correlation", + "title": "1 - Feature Engineering", + "section": "Reduce correlation ", + "text": "Reduce correlation \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-1", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps", + "title": "1 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#brier-score", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#brier-score", - "title": "4 - Evaluating models", - "section": "Brier score", - "text": "Brier score\nWhat if we don’t turn predicted probabilities into class predictions?\n\nThe Brier score is analogous to the mean squared error in regression models:\n\\[\nBrier_{class} = \\frac{1}{N}\\sum_{i=1}^N\\sum_{k=1}^C (y_{ik} - \\hat{p}_{ik})^2\n\\]" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps-1", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps-1", + "title": "1 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n embed::step_umap(all_numeric_predictors(), outcome = avg_price_per_room)\n\n\nA fancy machine learning supervised dimension reduction technique…\n\nNote that this uses the outcome, and it is from an extension package" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#brier-score-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#brier-score-1", - "title": "4 - Evaluating models", - "section": "Brier score", - "text": "Brier score\n\naugment(forested_fit, new_data = forested_train) |> \n brier_class(truth = forested, .pred_Yes) \n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.113\n\n\nSmaller values are better, for binary classification the “bad model threshold” is about 0.25." + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps-2", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#other-possible-steps-2", + "title": "1 - Feature Engineering", + "section": "Other possible steps ", + "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors()) %>% \n step_normalize(all_numeric_predictors()) %>% \n step_spline_natural(year_day, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#separation-vs-calibration", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#separation-vs-calibration", - "title": "4 - Evaluating models", - "section": "Separation vs calibration", - "text": "Separation vs calibration\n\n\nThe ROC captures separation.\n\n\n\n\n\n\n\n\n\n\nThe Brier score captures calibration.\n\n\n\n\n\n\n\n\n\n\n\n\nGood separation: the densities don’t overlap.\nGood calibration: the calibration line follows the diagonal.\n\nCalibration plot: We bin observations according to predicted probability. In the bin for 20%-30% predicted prob, we should see an event rate of ~25% if the model is well-calibrated." + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-2", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-2", + "title": "1 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\n\nCreate a recipe() for the hotel data to:\n\nuse a Yeo-Johnson (YJ) transformation on lead_time\nconvert factors to indicator variables\nremove zero-variance variables\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-1", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting", - "text": "Dangers of overfitting" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#minimal-recipe", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#minimal-recipe", + "title": "1 - Feature Engineering", + "section": "Minimal recipe ", + "text": "Minimal recipe \n\nhotel_indicators <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_YeoJohnson(lead_time) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-2", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-2", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️", - "text": "Dangers of overfitting ⚠️" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#measuring-performance", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#measuring-performance", + "title": "1 - Feature Engineering", + "section": "Measuring Performance ", + "text": "Measuring Performance \nWe’ll compute two measures: mean absolute error and the coefficient of determination (a.k.a \\(R^2\\)).\n\\[\\begin{align}\nMAE &= \\frac{1}{n}\\sum_{i=1}^n |y_i - \\hat{y}_i| \\notag \\\\\nR^2 &= cor(y_i, \\hat{y}_i)^2\n\\end{align}\\]\nThe focus will be on MAE for parameter optimization. We’ll use a metric set to compute these:\n\nreg_metrics <- metric_set(mae, rsq)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-3", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-3", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\nforested_fit |>\n augment(forested_train)\n#> # A tibble: 8,749 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness roughness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 0.931 0.0690 Yes 1997 66 82 10\n#> 2 Yes 0.983 0.0172 No 1997 284 -99 58\n#> 3 Yes 0.960 0.0401 Yes 2022 130 86 15\n#> 4 Yes 0.870 0.130 Yes 2021 202 -55 3\n#> 5 Yes 0.823 0.177 Yes 1995 75 -89 1\n#> 6 Yes 0.758 0.242 No 1995 110 -53 5\n#> 7 Yes 0.823 0.177 Yes 2022 111 73 12\n#> 8 No 0.467 0.533 Yes 1997 230 96 14\n#> 9 Yes 0.983 0.0172 Yes 2002 160 -88 13\n#> 10 Yes 0.871 0.129 Yes 2020 39 9 6\n#> # ℹ 8,739 more rows\n#> # ℹ 14 more variables: tree_no_tree <fct>, dew_temp <dbl>, precip_annual <dbl>,\n#> # temp_annual_mean <dbl>, temp_annual_min <dbl>, temp_annual_max <dbl>,\n#> # temp_january_min <dbl>, vapor_min <dbl>, vapor_max <dbl>,\n#> # canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>, county <fct>\n\nWe call this “resubstitution” or “repredicting the training set”" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#using-a-workflow", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#using-a-workflow", + "title": "1 - Feature Engineering", + "section": "Using a workflow ", + "text": "Using a workflow \n\nset.seed(9)\n\nhotel_lm_wflow <-\n workflow() %>%\n add_recipe(hotel_indicators) %>%\n add_model(linear_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nhotel_lm_res <-\n hotel_lm_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_lm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.3 10 0.199 Preprocessor1_Model1\n#> 2 rsq standard 0.874 10 0.00400 Preprocessor1_Model1" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-4", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-4", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n\nWe call this a “resubstitution estimate”" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-3", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#your-turn-3", + "title": "1 - Feature Engineering", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-5", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-5", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#holdout-predictions", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#holdout-predictions", + "title": "1 - Feature Engineering", + "section": "Holdout predictions ", + "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nlm_val_pred <- collect_predictions(hotel_lm_res)\nlm_val_pred %>% slice(1:7)\n#> # A tibble: 7 × 5\n#> id .pred .row avg_price_per_room .config \n#> <chr> <dbl> <int> <dbl> <chr> \n#> 1 Fold01 62.1 20 40 Preprocessor1_Model1\n#> 2 Fold01 48.0 28 54 Preprocessor1_Model1\n#> 3 Fold01 64.6 45 50 Preprocessor1_Model1\n#> 4 Fold01 45.8 49 42 Preprocessor1_Model1\n#> 5 Fold01 45.8 61 49 Preprocessor1_Model1\n#> 6 Fold01 30.0 66 40 Preprocessor1_Model1\n#> 7 Fold01 38.8 88 49 Preprocessor1_Model1" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-6", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-6", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.851\n\n\n\nforested_fit |>\n augment(forested_test) |>\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.711\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#calibration-plot", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#calibration-plot", + "title": "1 - Feature Engineering", + "section": "Calibration Plot ", + "text": "Calibration Plot \n\nlibrary(probably)\n\ncal_plot_regression(hotel_lm_res, alpha = 1 / 5)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-2", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-2", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse augment() and a metric function to compute a classification metric like brier_class().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", + "title": "1 - Feature Engineering", + "section": "What do we do with the agent and company data?", + "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 unique companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor (in the extra materials)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-7", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#dangers-of-overfitting-7", - "title": "4 - Evaluating models", - "section": "Dangers of overfitting ⚠️ ", - "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit |>\n augment(forested_train) |>\n brier_class(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.113\n\n\n\nforested_fit |>\n augment(forested_test) |>\n brier_class(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.208\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#per-agent-statistics", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#per-agent-statistics", + "title": "1 - Feature Engineering", + "section": "Per-agent statistics", + "text": "Per-agent statistics" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#collapsing-factor-levels", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#collapsing-factor-levels", + "title": "1 - Feature Engineering", + "section": "Collapsing factor levels ", + "text": "Collapsing factor levels \nThere is a recipe step that will redefine factor levels based on their frequency in the training set:\n\nhotel_other_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>% \n step_YeoJohnson(lead_time) %>%\n step_other(agent, threshold = 0.001) %>%\n step_dummy(all_nominal_predictors()) %>%\n step_zv(all_predictors())\n\nUsing this code, 34 agents (out of 98) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-1", - "title": "4 - Evaluating models", - "section": "Cross-validation", - "text": "Cross-validation" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#does-othering-help", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#does-othering-help", + "title": "1 - Feature Engineering", + "section": "Does othering help? ", + "text": "Does othering help? \n\nhotel_other_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hotel_other_rec)\n\nhotel_other_res <-\n hotel_other_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.4 10 0.205 Preprocessor1_Model1\n#> 2 rsq standard 0.874 10 0.00417 Preprocessor1_Model1\n\nAabout the same MAE and much faster to complete.\nNow let’s look at a more sophisticated tool called effect feature hashing." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-3", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-3", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing", + "title": "1 - Feature Engineering", + "section": "Feature Hashing", + "text": "Feature Hashing\nBetween agent and company, simple dummy variables would create 198 new columns (that are mostly zeros).\nAnother option is to have a binary indicator that combines some levels of these variables.\nFeature hashing (for more see FES, SMLTAR, and TMwR):\n\nuses the character values of the levels\nconverts them to integer hash values\nuses the integers to assign them to a specific indicator column." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-2", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-2", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(forested_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7874/875]> Fold01\n#> 2 <split [7874/875]> Fold02\n#> 3 <split [7874/875]> Fold03\n#> 4 <split [7874/875]> Fold04\n#> 5 <split [7874/875]> Fold05\n#> 6 <split [7874/875]> Fold06\n#> 7 <split [7874/875]> Fold07\n#> 8 <split [7874/875]> Fold08\n#> 9 <split [7874/875]> Fold09\n#> 10 <split [7875/874]> Fold10" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-1", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-1", + "title": "1 - Feature Engineering", + "section": "Feature Hashing", + "text": "Feature Hashing\nSuppose we want to use 32 indicator variables for agent.\nFor a agent with value “Max_Kuhn”, a hashing function converts it to an integer (say 210397726).\nTo assign it to one of the 32 columns, we would use modular arithmetic to assign it to a column:\n\n# For \"Max_Kuhn\" put a '1' in column: \n210397726 %% 32\n#> [1] 30\n\nHash functions are meant to emulate randomness." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-3", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-3", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWhat is in this?\n\nforested_folds <- vfold_cv(forested_train)\nforested_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <7874/875/8749>\n\n\nTalk about a list column, storing non-atomic types in dataframe" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-pros", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-pros", + "title": "1 - Feature Engineering", + "section": "Feature Hashing Pros", + "text": "Feature Hashing Pros\n\nThe procedure will automatically work on new values of the predictors.\nIt is fast.\n“Signed” hashes add a sign to help avoid aliasing." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-4", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-4", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \n\nvfold_cv(forested_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [6999/1750]> Fold1\n#> 2 <split [6999/1750]> Fold2\n#> 3 <split [6999/1750]> Fold3\n#> 4 <split [6999/1750]> Fold4\n#> 5 <split [7000/1749]> Fold5" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-cons", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-cons", + "title": "1 - Feature Engineering", + "section": "Feature Hashing Cons", + "text": "Feature Hashing Cons\n\nThere is no real logic behind which factor levels are combined.\nWe don’t know how many columns to add (more in the next section).\nSome columns may have all zeros.\nIf a indicator column is important to the model, we can’t easily determine why.\n\n\nThe signed hash make it slightly more possible to differentiate between confounded levels" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-5", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#cross-validation-5", - "title": "4 - Evaluating models", - "section": "Cross-validation ", - "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nforested_folds <- vfold_cv(forested_train, v = 10)\nforested_folds\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7874/875]> Fold01\n#> 2 <split [7874/875]> Fold02\n#> 3 <split [7874/875]> Fold03\n#> 4 <split [7874/875]> Fold04\n#> 5 <split [7874/875]> Fold05\n#> 6 <split [7874/875]> Fold06\n#> 7 <split [7874/875]> Fold07\n#> 8 <split [7874/875]> Fold08\n#> 9 <split [7874/875]> Fold09\n#> 10 <split [7875/874]> Fold10\n\n\nSet the seed when creating resamples" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-in-recipes", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-in-recipes", + "title": "1 - Feature Engineering", + "section": "Feature Hashing in recipes ", + "text": "Feature Hashing in recipes \nThe textrecipes package has a step that can be added to the recipe:\n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())\n\nhotel_hash_wflow <-\n hotel_lm_wflow %>%\n update_recipe(hash_rec)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", - "title": "4 - Evaluating models", - "section": "Fit our model to the resamples", - "text": "Fit our model to the resamples\n\nforested_res <- fit_resamples(forested_wflow, forested_folds)\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]>" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-in-recipes-1", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#feature-hashing-in-recipes-1", + "title": "1 - Feature Engineering", + "section": "Feature Hashing in recipes ", + "text": "Feature Hashing in recipes \n\nhotel_hash_res <-\n hotel_hash_wflow %>%\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_hash_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.5 10 0.256 Preprocessor1_Model1\n#> 2 rsq standard 0.872 10 0.00395 Preprocessor1_Model1\n\nAbout the same performance but now we can handle new values." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nforested_res |>\n collect_metrics()\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.704 10 0.00653 Preprocessor1_Model1\n#> 2 brier_class binary 0.214 10 0.00349 Preprocessor1_Model1\n#> 3 roc_auc binary 0.692 10 0.00496 Preprocessor1_Model1\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nWe can reliably measure performance using only the training data 🎉" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#debugging-a-recipe", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#debugging-a-recipe", + "title": "1 - Feature Engineering", + "section": "Debugging a recipe", + "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. hash_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#comparing-metrics", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#comparing-metrics", - "title": "4 - Evaluating models", - "section": "Comparing metrics ", - "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\nforested_res |>\n collect_metrics() |> \n select(.metric, mean, n)\n#> # A tibble: 3 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.704 10\n#> 2 brier_class 0.214 10\n#> 3 roc_auc 0.692 10\n\n\nThe ROC AUC previously was\n\n0.88 for the training set\n0.7 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#example", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#example", + "title": "1 - Feature Engineering", + "section": "Example ", + "text": "Example \n\nhash_rec_fit <- prep(hash_rec)\n\n# Get the transformation coefficient\ntidy(hash_rec_fit, number = 1)\n\n# Get the processed data\nbake(hash_rec_fit, hotel_tr %>% slice(1:3), contains(\"_agent_\"))" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-1", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_forested <- control_resamples(save_pred = TRUE)\nforested_res <- fit_resamples(forested_wflow, forested_folds, control = ctrl_forested)\n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]> <tibble>" + "objectID": "archive/2023-07-nyr/advanced-01-feature-engineering.html#more-on-recipes", + "href": "archive/2023-07-nyr/advanced-01-feature-engineering.html#more-on-recipes", + "title": "1 - Feature Engineering", + "section": "More on recipes", + "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-2", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-2", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\n# Save the assessment set results\nforested_preds <- collect_predictions(forested_res)\nforested_preds\n#> # A tibble: 8,749 × 7\n#> .pred_class .pred_Yes .pred_No id .row forested .config \n#> <fct> <dbl> <dbl> <chr> <int> <fct> <chr> \n#> 1 Yes 0.953 0.0473 Fold01 2 No Preprocessor1_Model1\n#> 2 Yes 0.6 0.4 Fold01 6 No Preprocessor1_Model1\n#> 3 Yes 0.848 0.152 Fold01 7 Yes Preprocessor1_Model1\n#> 4 Yes 0.941 0.0588 Fold01 36 Yes Preprocessor1_Model1\n#> 5 Yes 0.895 0.105 Fold01 38 No Preprocessor1_Model1\n#> 6 Yes 1 0 Fold01 58 Yes Preprocessor1_Model1\n#> 7 No 0.187 0.812 Fold01 69 No Preprocessor1_Model1\n#> 8 Yes 0.905 0.0952 Fold01 71 Yes Preprocessor1_Model1\n#> 9 Yes 0.907 0.0930 Fold01 74 Yes Preprocessor1_Model1\n#> 10 Yes 0.904 0.0962 Fold01 80 Yes Preprocessor1_Model1\n#> # ℹ 8,739 more rows" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#previously---setup", + "href": "archive/2023-07-nyr/advanced-03-racing.html#previously---setup", + "title": "3 - Grid Search via Racing", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(modeldatatoo)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\nset.seed(295)\nhotel_rates <- \n data_hotel_rates() %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date_num, -arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-3", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-3", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nforested_preds |> \n group_by(id) |>\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.729\n#> 2 Fold02 accuracy binary 0.685\n#> 3 Fold03 accuracy binary 0.687\n#> 4 Fold04 accuracy binary 0.711\n#> 5 Fold05 accuracy binary 0.736\n#> 6 Fold06 accuracy binary 0.670\n#> 7 Fold07 accuracy binary 0.699\n#> 8 Fold08 accuracy binary 0.710\n#> 9 Fold09 accuracy binary 0.699\n#> 10 Fold10 accuracy binary 0.719\n#> # ℹ 20 more rows" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#previously---data-usage", + "href": "archive/2023-07-nyr/advanced-03-racing.html#previously---data-usage", + "title": "3 - Grid Search via Racing", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_tr <- training(hotel_split)\nhotel_te <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_tr, strata = avg_price_per_room)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#where-are-the-fitted-models", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#where-are-the-fitted-models", - "title": "4 - Evaluating models", - "section": "Where are the fitted models? ", - "text": "Where are the fitted models? \n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7874/875]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7874/875]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7874/875]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7874/875]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7874/875]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7874/875]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7874/875]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7874/875]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7874/875]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7875/874]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#previously---boosting-model", + "href": "archive/2023-07-nyr/advanced-03-racing.html#previously---boosting-model", + "title": "3 - Grid Search via Racing", + "section": "Previously - Boosting Model ", + "text": "Previously - Boosting Model \n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_tr) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\")\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#bootstrapping", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#bootstrapping", - "title": "4 - Evaluating models", - "section": "Bootstrapping", - "text": "Bootstrapping" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#making-grid-search-more-efficient", + "href": "archive/2023-07-nyr/advanced-03-racing.html#making-grid-search-more-efficient", + "title": "3 - Grid Search via Racing", + "section": "Making Grid Search More Efficient", + "text": "Making Grid Search More Efficient\nIn the last section, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\nAlso, for some models, we can fit far fewer models than the number that are being evaluated.\n\nFor boosting, a model with X trees can often predict on candidates with less than X trees.\n\nBoth of these methods can lead to enormous speed-ups." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#bootstrapping-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#bootstrapping-1", - "title": "4 - Evaluating models", - "section": "Bootstrapping ", - "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(forested_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [8749/3218]> Bootstrap01\n#> 2 <split [8749/3264]> Bootstrap02\n#> 3 <split [8749/3220]> Bootstrap03\n#> 4 <split [8749/3208]> Bootstrap04\n#> 5 <split [8749/3230]> Bootstrap05\n#> 6 <split [8749/3197]> Bootstrap06\n#> 7 <split [8749/3193]> Bootstrap07\n#> 8 <split [8749/3226]> Bootstrap08\n#> 9 <split [8749/3243]> Bootstrap09\n#> 10 <split [8749/3233]> Bootstrap10\n#> # ℹ 15 more rows" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#model-racing", + "href": "archive/2023-07-nyr/advanced-03-racing.html#model-racing", + "title": "3 - Grid Search via Racing", + "section": "Model Racing", + "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models but only for a few resamples.\nDetermine which candidates have a low probability of being selected.\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain)\n\nThis can result in fitting a small number of models." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game---status-update", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game---status-update", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#discarding-candidates", + "href": "archive/2023-07-nyr/advanced-03-racing.html#discarding-candidates", + "title": "3 - Grid Search via Racing", + "section": "Discarding Candidates", + "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large difference between resamples in the results." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-4", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-4", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the functions)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#resampling-results-non-racing", + "href": "archive/2023-07-nyr/advanced-03-racing.html#resampling-results-non-racing", + "title": "3 - Grid Search via Racing", + "section": "Resampling Results (Non-Racing)", + "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#monte-carlo-cross-validation", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#monte-carlo-cross-validation", - "title": "4 - Evaluating models", - "section": "Monte Carlo Cross-Validation ", - "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(forested_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6561/2188]> Resample01\n#> 2 <split [6561/2188]> Resample02\n#> 3 <split [6561/2188]> Resample03\n#> 4 <split [6561/2188]> Resample04\n#> 5 <split [6561/2188]> Resample05\n#> 6 <split [6561/2188]> Resample06\n#> 7 <split [6561/2188]> Resample07\n#> 8 <split [6561/2188]> Resample08\n#> 9 <split [6561/2188]> Resample09\n#> 10 <split [6561/2188]> Resample10" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#are-candidates-different", + "href": "archive/2023-07-nyr/advanced-03-racing.html#are-candidates-different", + "title": "3 - Grid Search via Racing", + "section": "Are Candidates Different?", + "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval is (0.99, 2.8), indicating that candidate number 2 has smaller error.\nWhat if we were to compare each model candidate to the current best at each resample?\nOne shows superiority when 4 resamples have been evaluated." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#validation-set", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#validation-set", - "title": "4 - Evaluating models", - "section": "Validation set ", - "text": "Validation set \n\nset.seed(853)\nforested_val_split <- initial_validation_split(forested)\nvalidation_set(forested_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [4264/1421]> validation\n\n\nA validation set is just another type of resample" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#evaluating-differences-in-candidates", + "href": "archive/2023-07-nyr/advanced-03-racing.html#evaluating-differences-in-candidates", + "title": "3 - Grid Search via Racing", + "section": "Evaluating Differences in Candidates", + "text": "Evaluating Differences in Candidates" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#random-forest-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#random-forest-1", - "title": "4 - Evaluating models", - "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", - "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this later!), as long as there are enough trees" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#interim-analysis-of-results", + "href": "archive/2023-07-nyr/advanced-03-racing.html#interim-analysis-of-results", + "title": "3 - Grid Search via Racing", + "section": "Interim Analysis of Results", + "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order.\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#create-a-random-forest-model", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#create-a-random-forest-model", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#racing", + "href": "archive/2023-07-nyr/advanced-03-racing.html#racing", + "title": "3 - Grid Search via Racing", + "section": "Racing ", + "text": "Racing \n\n# Let's use a larger grid\nset.seed(8945)\nlgbm_grid <- \n lgbm_param %>% \n grid_latin_hypercube(size = 50)\n\nlibrary(finetune)\n\nset.seed(9)\nlgbm_race_res <-\n lgbm_wflow %>%\n tune_race_anova(\n resamples = hotel_rs,\n grid = lgbm_grid, \n metrics = reg_metrics\n )\n\nThe syntax and helper functions are extremely similar to those shown for tune_grid()." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#create-a-random-forest-model-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#create-a-random-forest-model-1", - "title": "4 - Evaluating models", - "section": "Create a random forest model ", - "text": "Create a random forest model \n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#racing-results", + "href": "archive/2023-07-nyr/advanced-03-racing.html#racing-results", + "title": "3 - Grid Search via Racing", + "section": "Racing Results ", + "text": "Racing Results \n\nshow_best(lgbm_race_res, metric = \"mae\")\n#> # A tibble: 2 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1014 5 0.0791 35 181 mae standard 10.3 10 0.202 Preprocessor06_Model1\n#> 2 1516 7 0.0421 176 12 mae standard 10.4 10 0.200 Preprocessor42_Model1" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-5", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#your-turn-5", - "title": "4 - Evaluating models", + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#racing-results-1", + "href": "archive/2023-07-nyr/advanced-03-racing.html#racing-results-1", + "title": "3 - Grid Search via Racing", + "section": "Racing Results ", + "text": "Racing Results \n\n\nOnly 170 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_race_res) + \n scale_x_continuous(breaks = pretty_breaks())" + }, + { + "objectID": "archive/2023-07-nyr/advanced-03-racing.html#your-turn", + "href": "archive/2023-07-nyr/advanced-03-racing.html#your-turn", + "title": "3 - Grid Search via Racing", "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" + "text": "Your turn\n\nRun tune_race_anova() with a different seed.\nDid you get the same or similar results?\n\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-4", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#evaluating-model-performance-4", - "title": "4 - Evaluating models", - "section": "Evaluating model performance ", - "text": "Evaluating model performance \n\nctrl_forested <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, forested_folds, control = ctrl_forested)\ncollect_metrics(rf_res)\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.752 10 0.00469 Preprocessor1_Model1\n#> 2 brier_class binary 0.167 10 0.00317 Preprocessor1_Model1\n#> 3 roc_auc binary 0.757 10 0.0100 Preprocessor1_Model1" + "objectID": "archive/2023-07-nyr/annotations.html#section", + "href": "archive/2023-07-nyr/annotations.html#section", + "title": "Annotations", + "section": "👀", + "text": "👀\nThis page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game---status-update-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game---status-update-1", - "title": "4 - Evaluating models", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2023-07-nyr/annotations.html#the-initial-split", + "href": "archive/2023-07-nyr/annotations.html#the-initial-split", + "title": "Annotations", + "section": "The initial split", + "text": "The initial split\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-final-fit", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-final-fit", - "title": "4 - Evaluating models", - "section": "The final fit ", - "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# forested_split has train + test info\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [8749/2188]> train/test split <tibble> <tibble> <tibble> <workflow>" + "objectID": "archive/2023-07-nyr/annotations.html#what-is-wrong-with-this", + "href": "archive/2023-07-nyr/annotations.html#what-is-wrong-with-this", + "title": "Annotations", + "section": "What is wrong with this?", + "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.761 Preprocessor1_Model1\n#> 2 roc_auc binary 0.761 Preprocessor1_Model1\n#> 3 brier_class binary 0.162 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" + "objectID": "archive/2023-07-nyr/annotations.html#where-are-the-fitted-models", + "href": "archive/2023-07-nyr/annotations.html#where-are-the-fitted-models", + "title": "Annotations", + "section": "Where are the fitted models?", + "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit-1", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit-1", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 2,188 × 7\n#> .pred_class .pred_Yes .pred_No id .row forested .config \n#> <fct> <dbl> <dbl> <chr> <int> <fct> <chr> \n#> 1 Yes 0.964 0.0363 train/test split 4 Yes Preprocessor1…\n#> 2 Yes 0.793 0.207 train/test split 8 Yes Preprocessor1…\n#> 3 Yes 0.815 0.185 train/test split 10 Yes Preprocessor1…\n#> 4 Yes 0.852 0.148 train/test split 19 Yes Preprocessor1…\n#> 5 Yes 0.838 0.162 train/test split 23 Yes Preprocessor1…\n#> 6 Yes 0.552 0.448 train/test split 28 No Preprocessor1…\n#> 7 Yes 0.831 0.169 train/test split 34 Yes Preprocessor1…\n#> 8 Yes 0.614 0.386 train/test split 35 No Preprocessor1…\n#> 9 Yes 0.975 0.0251 train/test split 38 Yes Preprocessor1…\n#> 10 Yes 0.944 0.0561 train/test split 40 Yes Preprocessor1…\n#> # ℹ 2,178 more rows" + "objectID": "archive/2023-07-nyr/annotations.html#validation-set", + "href": "archive/2023-07-nyr/annotations.html#validation-set", + "title": "Annotations", + "section": "Validation set", + "text": "Validation set\nThe upcoming version of the rsample package (1.2.0) will have a new set of functions specific to validation sets. They will allow you to make an initial three-way split and still use a validation set with the tune package." }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit-2", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#what-is-in-final_fit-2", - "title": "4 - Evaluating models", - "section": "What is in final_fit? ", - "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 8749 \n#> Number of independent variables: 18 \n#> Mtry: 4 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.1669661\n\n\nUse this for prediction on new data, like for deploying" + "objectID": "archive/2023-07-nyr/annotations.html#update-parameter-ranges", + "href": "archive/2023-07-nyr/annotations.html#update-parameter-ranges", + "title": "Annotations", + "section": "Update parameter ranges", + "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_spline_natural(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" }, { - "objectID": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game", - "href": "archive/2025-08-nyr/slides/intro-04-evaluating-models.html#the-whole-game", - "title": "4 - Evaluating models", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-07-nyr/annotations.html#early-stopping-for-boosted-trees", + "href": "archive/2023-07-nyr/annotations.html#early-stopping-for-boosted-trees", + "title": "Annotations", + "section": "Early stopping for boosted trees", + "text": "Early stopping for boosted trees\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the lightgbm package.\nbonsai has its own wrapper around (lightgbm::lgb.train()) called bonsai::train_lightgbm(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the lightgbm entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for lightgbm)." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia", - "title": "2 - Your data budget", - "section": "Data on forests in Georgia", - "text": "Data on forests in Georgia\n\n\n\nThe U.S. Forest Service maintains ML models to predict whether a plot of land is “forested.”\nThis classification is important for all sorts of research, legislation, and land management purposes.\nPlots are typically remeasured every 10 years and this dataset contains the most recent measurement per plot.\nType ?forested_ga to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/251793/forest-mountain" + "objectID": "archive/2023-07-nyr/annotations.html#per-agent-statistics", + "href": "archive/2023-07-nyr/annotations.html#per-agent-statistics", + "title": "Annotations", + "section": "Per-agent statistics", + "text": "Per-agent statistics\nThe effect encoding method essentially takes the effect of a variable, like agent, and makes a data column for that effect. In our example, affect of the agent on the ADR is quantified by a model and then added as a data column to be used in the model.\nSuppose agent Max has a single reservation in the data and it had an ADR of €200. If we used a naive estimate for Max’s effect, the model is being told that Max should always produce an effect of €200. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned agent Davis, who has taken 250 reservations with an average ADR of €100. Davis’s mean is more predictive because it is estimated with better data (i.e., more total reservations). Partial pooling leverages the entire data set and can borrow strength from all of the agents. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a agent’s data is of good quality, the partial pooling effect estimate is closer to the raw mean. Max’s data is not great and is “shrunk” towards the center of the overall average. Since there is so little known about Max’s reservation history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nSince this example has a numeric outcome, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia-1", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia-1", - "title": "2 - Your data budget", - "section": "Data on forests in Georgia", - "text": "Data on forests in Georgia\n\n\n\nN = 10937 plots of land, one from each of 10937 6000-acre hexagons in Georgia.\nA nominal outcome, forested, with levels \"Yes\" and \"No\", measured “on-the-ground.”\n18 remotely-sensed and easily-accessible predictors:\n\nnumeric variables based on weather and topography.\nnominal variables based on classifications from other governmental orgs.\n\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/67614/forest\n\n\n\nThose nominal variables are classifications similar to “forested” but from other agencies. e.g. land_type is from the European Space Agency, and is a remotely-sensed 3-class distribution based on predictions for how the land is used. (The data also includes a county variable, which is not itself from a model.)" + "objectID": "archive/2023-07-nyr/annotations.html#agent-effects", + "href": "archive/2023-07-nyr/annotations.html#agent-effects", + "title": "Annotations", + "section": "Agent effects", + "text": "Agent effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage though." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#checklist-for-predictors", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#checklist-for-predictors", - "title": "2 - Your data budget", - "section": "Checklist for predictors", - "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?\n\n\n\nre: ethics – what issues might arise from releasing the true lat and lon? In reality, these lat and lon are slightly jittered to help ensure trust with landowners who allow surveyers to come take measurements." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#chicago-l-train-data", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#chicago-l-train-data", + "title": "Case Study on Transportation", + "section": "Chicago L-Train data", + "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia-2", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-on-forests-in-georgia-2", - "title": "2 - Your data budget", - "section": "Data on forests in Georgia", - "text": "Data on forests in Georgia\n\nlibrary(tidymodels)\nlibrary(forested)\n\nforested_ga\n#> # A tibble: 10,937 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 2007 14 0 0 No tree 13.9\n#> 2 Yes 2007 66 -53 10 Tree 13.8\n#> 3 Yes 2006 59 -82 6 No tree 13.5\n#> 4 Yes 2007 116 -78 20 Tree 12.3\n#> 5 Yes 2006 283 63 13 Tree 10.0\n#> 6 Yes 2007 250 63 14 Tree 10.8\n#> 7 Yes 2007 58 31 1 No tree 13.8\n#> 8 Yes 2023 140 56 11 Tree 12.2\n#> 9 Yes 2024 118 72 17 Tree 12.2\n#> 10 Yes 2024 217 -46 13 Tree 12.4\n#> # ℹ 10,927 more rows\n#> # ℹ 12 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>, county <fct>" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#predictors", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#predictors", + "title": "Case Study on Transportation", + "section": "Predictors", + "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#l-train-locations", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#l-train-locations", + "title": "Case Study on Transportation", + "section": "L Train Locations", + "text": "L Train Locations" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-1", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-1", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#your-turn-explore-the-data", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#your-turn-explore-the-data", + "title": "Case Study on Transportation", + "section": "Your turn: Explore the Data", + "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n−+\n05:00" + }, + { + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#splitting-with-chicago-data", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#splitting-with-chicago-data", + "title": "Case Study on Transportation", + "section": "Splitting with Chicago data ", + "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-2", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-2", - "title": "2 - Your data budget", - "section": "Data splitting and spending", - "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#time-series-resampling", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#time-series-resampling", + "title": "Case Study on Transportation", + "section": "Time series resampling", + "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#rolling-forecast-origin-resampling", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#rolling-forecast-origin-resampling", + "title": "Case Study on Transportation", + "section": "Rolling forecast origin resampling", + "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-initial-split", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-initial-split", - "title": "2 - Your data budget", - "section": "The initial split ", - "text": "The initial split \n\nset.seed(123)\nforested_split <- initial_split(forested_ga)\nforested_split\n#> <Training/Testing/Total>\n#> <8202/2735/10937>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#what-is-set.seed", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#what-is-set.seed", - "title": "2 - Your data budget", - "section": "What is set.seed()?", - "text": "What is set.seed()?\nTo create that split of the data, R generates “pseudo-random” numbers: while they are made to behave like random numbers, their generation is deterministic given a “seed”.\nThis allows us to reproduce results by setting that seed.\nWhich seed you pick doesn’t matter, as long as you don’t try a bunch of seeds and pick the one that gives you the best performance." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-1", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-1", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#accessing-the-data", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#accessing-the-data", - "title": "2 - Your data budget", - "section": "Accessing the data ", - "text": "Accessing the data \n\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-2", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-2", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-training-set", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-training-set", - "title": "2 - Your data budget", - "section": "The training set", - "text": "The training set\n\nforested_train\n#> # A tibble: 8,202 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 1997 66 82 10 Tree 12.2 \n#> 2 No 1997 284 -99 58 Tree 10.3 \n#> 3 Yes 2022 130 86 15 Tree 11.8 \n#> 4 Yes 2021 202 -55 3 Tree 10.7 \n#> 5 Yes 1995 75 -89 1 Tree 13.8 \n#> 6 No 1995 110 -53 5 Tree 12.4 \n#> 7 Yes 2022 111 73 12 Tree 11.5 \n#> 8 Yes 1997 230 96 14 Tree 9.98\n#> 9 Yes 2002 160 -88 13 Tree 11.1 \n#> 10 Yes 2020 39 9 6 Tree 13.9 \n#> # ℹ 8,192 more rows\n#> # ℹ 12 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>, county <fct>" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-3", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-3", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-test-set", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-test-set", - "title": "2 - Your data budget", - "section": "The test set ", - "text": "The test set \n🙈\n\nThere are 2735 rows and 19 columns in the test set." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-4", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#times-series-resampling-4", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn-1", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn-1", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#our-resampling-object", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#our-resampling-object", + "title": "Case Study on Transportation", + "section": "Our resampling object ", + "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-3", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#data-splitting-and-spending-3", - "title": "2 - Your data budget", - "section": "Data splitting and spending ", - "text": "Data splitting and spending \n\nset.seed(123)\nforested_split <- initial_split(forested_ga, prop = 0.8)\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)\n\nnrow(forested_train)\n#> [1] 8749\nnrow(forested_test)\n#> [1] 2188" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#feature-engineering-with-recipes", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#feature-engineering-with-recipes", + "title": "Case Study on Transportation", + "section": "Feature engineering with recipes ", + "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn-2", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#your-turn-2", - "title": "2 - Your data budget", - "section": "Your turn", - "text": "Your turn\n\nExplore the forested_train data on your own!\n\nWhat’s the distribution of the outcome, forested?\nWhat’s the distribution of numeric variables like precip_annual?\nHow does the distribution of forested differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe", + "title": "Case Study on Transportation", + "section": "A recipe", + "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-1", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-1", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = forested)) +\n geom_bar()" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-2", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-2", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = forested, fill = tree_no_tree)) +\n geom_bar()" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates-1", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates-1", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-3", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-3", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"identity\", alpha = .7)" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates-2", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---work-with-dates-2", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-4", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-4", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"fill\")" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---remove-constant-columns", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---remove-constant-columns", + "title": "Case Study on Transportation", + "section": "A recipe - remove constant columns ", + "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE) %>% \n step_zv(all_nominal_predictors())" }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-5", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#section-5", - "title": "2 - Your data budget", - "section": "", - "text": "forested_train |> \n ggplot(aes(x = lon, y = lat, col = forested)) +\n geom_point()" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---handle-correlations", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#a-recipe---handle-correlations", + "title": "Case Study on Transportation", + "section": "A recipe - handle correlations ", + "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." }, { - "objectID": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-whole-game---status-update", - "href": "archive/2025-08-nyr/slides/intro-02-data-budget.html#the-whole-game---status-update", - "title": "2 - Your data budget", - "section": "The whole game - status update", - "text": "The whole game - status update" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#make-some-models", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#make-some-models", + "title": "Case Study on Transportation", + "section": "Make some models ", + "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) %>% set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) %>% \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." }, { - "objectID": "archive/2025-08-nyr/slides/extras-vetiver.html#deploying-a-model", - "href": "archive/2025-08-nyr/slides/extras-vetiver.html#deploying-a-model", - "title": "Extras - Model deployment", - "section": "Deploying a model ", - "text": "Deploying a model \nWe have a decision tree, tree_fit, to model whether or not a plot of land in Washington is forested or not.\nHow do we use our model in production?\n\nlibrary(vetiver)\nv <- vetiver_model(tree_fit, \"forested\")\nv\n#> \n#> ── forested ─ <bundled_workflow> model for deployment \n#> A rpart classification modeling workflow using 19 features\n\nLearn more at https://vetiver.rstudio.com" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#process-them-on-the-resamples", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#process-them-on-the-resamples", + "title": "Case Study on Transportation", + "section": "Process them on the resamples", + "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set %>% \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" }, { - "objectID": "archive/2025-08-nyr/slides/extras-vetiver.html#deploy-your-model-1", - "href": "archive/2025-08-nyr/slides/extras-vetiver.html#deploy-your-model-1", - "title": "Extras - Model deployment", - "section": "Deploy your model ", - "text": "Deploy your model \nHow do we use our model in production?\n\nlibrary(plumber)\npr() |>\n vetiver_api(v)\n#> # Plumber router with 4 endpoints, 4 filters, and 1 sub-router.\n#> # Use `pr_run()` on this object to start the API.\n#> ├──[queryString]\n#> ├──[body]\n#> ├──[cookieParser]\n#> ├──[sharedSecret]\n#> ├──/logo\n#> │ │ # Plumber static router serving from directory: /Users/emilhvitfeldt/Library/R/arm64/4.5/library/vetiver\n#> ├──/metadata (GET)\n#> ├──/ping (GET)\n#> ├──/predict (POST)\n#> └──/prototype (GET)\n\nLearn more at https://vetiver.rstudio.com\n\nLive-code making a prediction" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#how-do-the-results-look", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#how-do-the-results-look", + "title": "Case Study on Transportation", + "section": "How do the results look?", + "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 31 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.798 0.104 16 recipe cubis… 1\n#> 2 pca_cubist Preprocessor3_Model3 mae 0.978 0.110 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor4_Model2 mae 0.983 0.122 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model1 mae 0.991 0.127 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor3_Model2 mae 0.991 0.113 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor2_Model2 mae 1.02 0.118 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor1_Model3 mae 1.05 0.134 16 recipe cubis… 7\n#> 8 basic_cubist Preprocessor1_Model8 mae 1.07 0.115 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model7 mae 1.07 0.112 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model6 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 21 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/extras-vetiver.html#your-turn", - "href": "archive/2025-08-nyr/slides/extras-vetiver.html#your-turn", - "title": "Extras - Model deployment", - "section": "Your turn", - "text": "Your turn\n\nRun the vetiver chunk in your .qmd.\nCheck out the automated visual documentation.\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#plot-the-results", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#plot-the-results", + "title": "Case Study on Transportation", + "section": "Plot the results ", + "text": "Plot the results \n\nautoplot(chi_res)" }, { - "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#previously---setup", - "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#previously---setup", - "title": "Extras - Effect Encodings", - "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#pull-out-specific-results", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#pull-out-specific-results", + "title": "Case Study on Transportation", + "section": "Pull out specific results ", + "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n autoplot()" }, { - "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#previously---data-usage", - "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#previously---data-usage", - "title": "Extras - Effect Encodings", - "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." }, { - "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#what-do-we-do-with-the-agent-and-company-data", - "title": "Extras - Effect Encodings", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-1", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-1", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#per-agent-statistics", - "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#per-agent-statistics", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-2", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-2", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#what-is-an-effect-encoding", - "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#what-is-an-effect-encoding", - "title": "Extras - Effect Encodings", - "section": "What is an effect encoding?", - "text": "What is an effect encoding?\nWe replace the qualitative’s predictor data with their effect on the outcome.\n\n\nData before:\n\nbefore\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <fct> <int>\n#> 1 52.7 cynthia_worsley 1\n#> 2 51.8 carlos_bryant 2\n#> 3 53.8 lance_hitchcock 3\n#> 4 51.8 lance_hitchcock 4\n#> 5 46.8 cynthia_worsley 5\n#> 6 54.7 charles_najera 6\n#> 7 46.8 cynthia_worsley 7\n\n\nData after:\n\nafter\n#> # A tibble: 7 × 3\n#> avg_price_per_room agent .row\n#> <dbl> <dbl> <int>\n#> 1 52.7 88.5 1\n#> 2 51.8 89.5 2\n#> 3 53.8 79.8 3\n#> 4 51.8 79.8 4\n#> 5 46.8 88.5 5\n#> 6 54.7 109. 6\n#> 7 46.8 88.5 7\n\n\nThe agent column is replaced with an estimate of the ADR." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-3", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-3", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#per-agent-statistics-again", - "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#per-agent-statistics-again", - "title": "Extras - Effect Encodings", - "section": "Per-agent statistics again", - "text": "Per-agent statistics again\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nGood statistical methods for estimating these means use partial pooling.\nPooling borrows strength across agents and shrinks extreme values towards the mean for agents with very few transations\nThe embed package has recipe steps for effect encodings.\n\n\n\nPartial pooling gives better estimates for agents with fewer reservations by shrinking the estimate to the overall ADR mean" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-4", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-4", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#partial-pooling", - "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#partial-pooling", - "title": "Extras - Effect Encodings", - "section": "Partial pooling", - "text": "Partial pooling" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-5", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#why-choose-just-one-final_fit-5", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#agent-effects", - "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#agent-effects", - "title": "Extras - Effect Encodings", - "section": "Agent effects ", - "text": "Agent effects \n\nlibrary(embed)\n\nhotel_effect_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_YeoJohnson(lead_time) |>\n step_lencode_mixed(agent, company, outcome = vars(avg_price_per_room)) |>\n step_dummy(all_nominal_predictors()) |>\n step_zv(all_predictors())\n\n\nIt is very important to appropriately validate the effect encoding step to make sure that we are not overfitting." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#building-a-model-stack", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#building-a-model-stack", + "title": "Case Study on Transportation", + "section": "Building a model stack ", + "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" }, { - "objectID": "archive/2025-08-nyr/slides/extras-effect-encodings.html#effect-encoding-results", - "href": "archive/2025-08-nyr/slides/extras-effect-encodings.html#effect-encoding-results", - "title": "Extras - Effect Encodings", - "section": "Effect encoding results ", - "text": "Effect encoding results \n\nhotel_effect_wflow <-\n workflow() |>\n add_model(linear_reg()) |> \n update_recipe(hotel_effect_rec)\n\nreg_metrics <- metric_set(mae, rsq)\n\nhotel_effect_res <-\n hotel_effect_wflow |>\n fit_resamples(hotel_rs, metrics = reg_metrics)\n\ncollect_metrics(hotel_effect_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 17.8 10 0.189 Preprocessor1_Model1\n#> 2 rsq standard 0.870 10 0.00357 Preprocessor1_Model1\n\nSlightly worse but it can handle new agents (if they occur)." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#start-the-stack-and-add-members", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#start-the-stack-and-add-members", + "title": "Case Study on Transportation", + "section": "Start the stack and add members ", + "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() %>% \n add_candidates(chi_res)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-06-wrapping-up.html#your-turn", - "href": "archive/2025-08-nyr/slides/advanced-06-wrapping-up.html#your-turn", - "title": "6 - Wrapping up", - "section": "Your turn", - "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#estimate-weights-for-each-candidate", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#estimate-weights-for-each-candidate", + "title": "Case Study on Transportation", + "section": "Estimate weights for each candidate ", + "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack)\n\nchi_stack_res\n#> # A tibble: 5 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.343\n#> 2 pca_cubist_3_2 cubist_rules 0.236\n#> 3 basic_cubist_1_4 cubist_rules 0.189\n#> 4 pca_lm_4_1 linear_reg 0.163\n#> 5 pca_cubist_3_3 cubist_rules 0.109" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-06-wrapping-up.html#resources-to-keep-learning", - "href": "archive/2025-08-nyr/slides/advanced-06-wrapping-up.html#resources-to-keep-learning", - "title": "6 - Wrapping up", - "section": "Resources to keep learning", - "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://tidymodels.aml4td.org/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#how-did-it-do", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#how-did-it-do", + "title": "Case Study on Transportation", + "section": "How did it do? ", + "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---setup", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---setup", - "title": "4 - Grid Search via Racing", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#what-does-it-use", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#what-does-it-use", + "title": "Case Study on Transportation", + "section": "What does it use? ", + "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---data-usage", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---data-usage", - "title": "4 - Grid Search via Racing", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <-\n initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#fit-the-required-candidate-models", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#fit-the-required-candidate-models", + "title": "Case Study on Transportation", + "section": "Fit the required candidate models", + "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---boosting-model", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#previously---boosting-model", - "title": "4 - Grid Search via Racing", - "section": "Previously - Boosting Model ", - "text": "Previously - Boosting Model \n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) |>\n step_dummy_hash(company, num_terms = tune(\"company hash\")) |>\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow |>\n extract_parameter_set_dials() |>\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#the-test-set-best-cubist-model", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#the-test-set-best-cubist-model", + "title": "Case Study on Transportation", + "section": "The test set: best Cubist model ", + "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n select_best()\n\ncubist_res <- \n chi_res %>% \n extract_workflow(\"pca_cubist\") %>% \n finalize_workflow(best_cubist) %>% \n last_fit(split = chi_split, metrics = metric_set(mae))" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#first-a-shameless-promotion", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#first-a-shameless-promotion", - "title": "4 - Grid Search via Racing", - "section": "First, a shameless promotion", - "text": "First, a shameless promotion" + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#the-test-set-stack-ensemble", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#the-test-set-stack-ensemble", + "title": "Case Study on Transportation", + "section": "The test set: stack ensemble", + "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) %>% \n bind_cols(chi_test)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#making-grid-search-more-efficient", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#making-grid-search-more-efficient", - "title": "4 - Grid Search via Racing", - "section": "Making Grid Search More Efficient", - "text": "Making Grid Search More Efficient\nIn the last section, we evaluated 250 models (25 candidates times 10 resamples).\nWe can make this go faster using parallel processing.\nAlso, for some models, we can fit far fewer models than the number that are being evaluated.\n\nFor boosting, a model with X trees can often predict on candidates with less than X trees.\n\nBoth of these methods can lead to enormous speed-ups." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#compare-the-results", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#compare-the-results", + "title": "Case Study on Transportation", + "section": "Compare the results ", + "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.670 Preprocessor1_Model1\n\nstack_pred %>% mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.689" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#model-racing", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#model-racing", - "title": "4 - Grid Search via Racing", - "section": "Model Racing", - "text": "Model Racing\nRacing is an old tool that we can use to go even faster.\n\nEvaluate all of the candidate models but only for a few resamples.\nDetermine which candidates have a low probability of being selected.\nEliminate poor candidates.\nRepeat with next resample (until no more resamples remain)\n\nThis can result in fitting a small number of models." + "objectID": "archive/2023-07-nyr/extras-transit-case-study.html#plot-the-test-set", + "href": "archive/2023-07-nyr/extras-transit-case-study.html#plot-the-test-set", + "title": "Case Study on Transportation", + "section": "Plot the test set ", + "text": "Plot the test set \n\n\nlibrary(probably)\ncubist_res %>% \n collect_predictions() %>% \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#discarding-candidates", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#discarding-candidates", - "title": "4 - Grid Search via Racing", - "section": "Discarding Candidates", - "text": "Discarding Candidates\nHow do we eliminate tuning parameter combinations?\nThere are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).\nHowever… there is typically a large difference between resamples in the results." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#workshop-policies", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#workshop-policies", + "title": "1 - Introduction", + "section": "Workshop policies", + "text": "Workshop policies\n\nPlease do not photograph people wearing red lanyards\nThere are gender-neutral bathrooms located are among the Grand Suite Bathrooms\nThere are two meditation/prayer rooms: Grand Suite 2A and 2B\nA lactation room is located in Grand Suite 1\nThe meditation/prayer and lactation rooms are open\nSun - Tue 7:30am - 7:00pm, Wed 8:00am - 6:00pm" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#resampling-results-non-racing", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#resampling-results-non-racing", - "title": "4 - Grid Search via Racing", - "section": "Resampling Results (Non-Racing)", - "text": "Resampling Results (Non-Racing)\n\n\nHere are some realistic (but simulated) examples of two candidate models.\nAn error estimate is measured for each of 10 resamples.\n\nThe lines connect resamples.\n\nThere is usually a significant resample-to-resample effect (rank corr: 0.83)." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#workshop-policies-1", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#workshop-policies-1", + "title": "1 - Introduction", + "section": "Workshop policies", + "text": "Workshop policies\n\nPlease review the code of conduct and COVID policies, which apply to all workshops: https://posit.co/code-of-conduct/.\nCoC site has info on how to report a problem (in person, email, phone)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#are-candidates-different", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#are-candidates-different", - "title": "4 - Grid Search via Racing", - "section": "Are Candidates Different?", - "text": "Are Candidates Different?\nOne way to evaluate these models is to do a paired t-test\n\nor a t-test on their differences matched by resamples\n\nWith \\(n = 10\\) resamples, the confidence interval for the difference in RMSE is (0.99, 2.8), indicating that candidate number 2 has smaller error." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#who-are-you", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML\nYou have used some tidymodels packages\nYou have some experience with evaluating statistical models using resampling techniques" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#evaluating-differences-in-candidates", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#evaluating-differences-in-candidates", - "title": "4 - Grid Search via Racing", - "section": "Evaluating Differences in Candidates", - "text": "Evaluating Differences in Candidates\n\n\nWhat if we were to have compared the candidates while we seqeuntially evaluated each resample?\n👉\n\nOne candidate shows superiority when 4 resamples have been evaluated." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#who-are-tidymodels", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nIjeamaka Anyene (Day 1) and Edgar Ruiz (Day 2) are TAing!\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#interim-analysis-of-results", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#interim-analysis-of-results", - "title": "4 - Grid Search via Racing", - "section": "Interim Analysis of Results", - "text": "Interim Analysis of Results\nOne version of racing uses a mixed model ANOVA to construct one-sided confidence intervals for each candidate versus the current best.\nAny candidates whose bound does not include zero are discarded. Here is an animation.\nThe resamples are analyzed in a random order (so set the seed).\n\nKuhn (2014) has examples and simulations to show that the method works.\nThe finetune package has functions tune_race_anova() and tune_race_win_loss()." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#asking-for-help", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#asking-for-help", + "title": "1 - Introduction", + "section": "Asking for help", + "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing", - "title": "4 - Grid Search via Racing", - "section": "Racing ", - "text": "Racing \n\n# Let's use a larger grid\nlgbm_grid <- \n lgbm_param |> \n grid_space_filling(size = 50)\n\nlibrary(finetune)\n\nset.seed(9)\nlgbm_race_res <-\n lgbm_wflow |>\n tune_race_anova(\n resamples = hotel_rs,\n grid = lgbm_grid, \n metrics = reg_metrics\n )\n\nThe syntax and helper functions are extremely similar to those shown for tune_grid()." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#section-2", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#section-2", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing-results", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing-results", - "title": "4 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\nshow_best(lgbm_race_res, metric = \"mae\")\n#> # A tibble: 2 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1347 5 0.0655 66 26 mae standard 9.64 10 0.173 Preprocessor34_Model1\n#> 2 980 8 0.0429 17 135 mae standard 9.76 10 0.164 Preprocessor25_Model1" + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#tentative-plan-for-this-workshop", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#tentative-plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Tentative plan for this workshop", + "text": "Tentative plan for this workshop\n\nFeature engineering with recipes\nModel optimization by tuning\n\nGrid search\nRacing\nIterative methods\n\nExtras (time permitting)\n\nEffect encodings\nA case study" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing-results-1", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#racing-results-1", - "title": "4 - Grid Search via Racing", - "section": "Racing Results ", - "text": "Racing Results \n\n\nOnly 171 models were fit (out of 500).\nselect_best() never considers candidate models that did not get to the end of the race.\nThere is a helper function to see how candidate models were removed from consideration.\n\n\nplot_race(lgbm_race_res) + \n scale_x_continuous(breaks = pretty_breaks())" + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#section-3", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#section-3", + "title": "1 - Introduction", + "section": "", + "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free):\nCheck the workshop channel on Discord for the link!" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-04-racing.html#your-turn", - "href": "archive/2025-08-nyr/slides/advanced-04-racing.html#your-turn", - "title": "4 - Grid Search via Racing", - "section": "Your turn", - "text": "Your turn\n\nRun tune_race_anova() with a different seed.\nDid you get the same or similar results?\n\n\n\n\n−+\n10:00" + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#lets-install-some-packages", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of RStudio Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\",\n \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\",\n \"splines2\", \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \n \"vetiver\", \"remotes\")\n\ninstall.packages(pkgs)\n\n\n\n Or log in to Posit Cloud\nLink in our Discord channel!" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#working-with-our-predictors", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#working-with-our-predictors", - "title": "2 - Feature Engineering", - "section": "Working with our predictors", - "text": "Working with our predictors\nWe might want to modify our predictors columns for a few reasons:\n\n\nThe model requires them in a different format (e.g. dummy variables for linear regression).\nThe model needs certain data qualities (e.g. same units for K-NN).\nThe outcome is better predicted when one or more columns are transformed in some way (a.k.a “feature engineering”).\n\n\n\nThe first two reasons are fairly predictable (next page).\nThe last one depends on your modeling problem." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#hotel-data", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#hotel-data", + "title": "1 - Introduction", + "section": "Hotel Data ", + "text": "Hotel Data \nWe’ll use data on hotels to predict the cost of a room.\nThe data are in the modeldata package. We’ll sample down the data and refactor some columns:\n\n\n\nlibrary(tidymodels)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#what-is-feature-engineering", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#what-is-feature-engineering", - "title": "2 - Feature Engineering", - "section": "What is feature engineering?", - "text": "What is feature engineering?\nThink of a feature as some representation of a predictor that will be used in a model.\n\nExample representations:\n\nInteractions\nPolynomial expansions/splines\nPrincipal component analysis (PCA) feature extraction\n\nThere are a lot of examples in Feature Engineering and Selection (FES) and Applied Machine Learning for Tabular Data (aml4td)." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#hotel-date-columns", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#hotel-date-columns", + "title": "1 - Introduction", + "section": "Hotel date columns", + "text": "Hotel date columns\n\nnames(hotel_rates)\n#> [1] \"avg_price_per_room\" \"lead_time\" \n#> [3] \"stays_in_weekend_nights\" \"stays_in_week_nights\" \n#> [5] \"adults\" \"children\" \n#> [7] \"babies\" \"meal\" \n#> [9] \"country\" \"market_segment\" \n#> [11] \"distribution_channel\" \"is_repeated_guest\" \n#> [13] \"previous_cancellations\" \"previous_bookings_not_canceled\"\n#> [15] \"reserved_room_type\" \"assigned_room_type\" \n#> [17] \"booking_changes\" \"agent\" \n#> [19] \"company\" \"days_in_waiting_list\" \n#> [21] \"customer_type\" \"required_car_parking_spaces\" \n#> [23] \"total_of_special_requests\" \"arrival_date_num\" \n#> [25] \"near_christmas\" \"near_new_years\" \n#> [27] \"historical_adr\"" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#example-dates", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#example-dates", - "title": "2 - Feature Engineering", - "section": "Example: Dates", - "text": "Example: Dates\nHow can we represent date columns for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#data-splitting-strategy", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#data-splitting-strategy", + "title": "1 - Introduction", + "section": "Data splitting strategy", + "text": "Data splitting strategy" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#general-definitions", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#general-definitions", - "title": "2 - Feature Engineering", - "section": "General definitions ", - "text": "General definitions \n\nData preprocessing steps allow your model to fit.\nFeature engineering steps help the model do the least work to predict the outcome as well as possible.\n\nThe recipes package can handle both!\n\nThese terms are often used interchangeably in the ML community but we want to distinguish them." + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#data-spending", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#data-spending", + "title": "1 - Introduction", + "section": "Data Spending ", + "text": "Data Spending \nLet’s split the data into a training set (75%) and testing set (25%) using stratification:\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#previously---setup", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#previously---setup", - "title": "2 - Feature Engineering", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\n\n# Add another package:\nlibrary(textrecipes)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#your-turn", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#your-turn", + "title": "1 - Introduction", + "section": "Your turn", + "text": "Your turn\nLet’s take some time and investigate the training data. The outcome is avg_price_per_room.\nAre there any interesting characteristics of the data?\n\n\n\n−+\n10:00" + }, + { + "objectID": "archive/2023-09-posit-conf/advanced-01-introduction.html#our-versions", + "href": "archive/2023-09-posit-conf/advanced-01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nR version 4.2.2 (2022-10-31), Quarto (1.4.104)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.2.1\n\n\nbroom\n1.0.5\n\n\ndials\n1.2.0\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.3\n\n\nembed\n1.1.2\n\n\nfinetune\n1.1.0\n\n\nggplot2\n3.4.3\n\n\nlightgbm\n3.3.5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nlme4\n1.1-34\n\n\nmodeldata\n1.2.0\n\n\nparsnip\n1.1.1\n\n\nplumber\n1.2.1\n\n\nprobably\n1.0.2\n\n\npurrr\n1.0.2\n\n\nranger\n0.15.1\n\n\nrecipes\n1.0.8\n\n\nremotes\n2.4.2.1\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nrpart\n4.1.19\n\n\nrpart.plot\n3.1.1\n\n\nrsample\n1.2.0\n\n\nrules\n1.0.2\n\n\nscales\n1.2.1\n\n\nsplines2\n0.5.1\n\n\nstacks\n1.0.2\n\n\ntext2vec\n0.6.3\n\n\ntextrecipes\n1.0.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.1.1\n\n\ntidyr\n1.3.0\n\n\ntune\n1.1.2\n\n\nvetiver\n0.2.4\n\n\nworkflows\n1.1.3\n\n\nworkflowsets\n1.0.1\n\n\nyardstick\n1.2.0" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#previously---data-usage", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#previously---data-usage", - "title": "2 - Feature Engineering", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\n\nWe’ll go from here and create a set of resamples to use for model assessments." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---setup", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---setup", + "title": "2 - Tuning Hyperparameters", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#resampling-strategy", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#resampling-strategy", - "title": "2 - Feature Engineering", - "section": "Resampling Strategy", - "text": "Resampling Strategy" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---data-usage", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---data-usage", + "title": "2 - Tuning Hyperparameters", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#resampling-strategy-1", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#resampling-strategy-1", - "title": "2 - Feature Engineering", - "section": "Resampling Strategy ", - "text": "Resampling Strategy \nWe’ll use simple 10-fold cross-validation (stratified sampling):\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)\nhotel_rs\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [3372/377]> Fold01\n#> 2 <split [3373/376]> Fold02\n#> 3 <split [3373/376]> Fold03\n#> 4 <split [3373/376]> Fold04\n#> 5 <split [3373/376]> Fold05\n#> 6 <split [3374/375]> Fold06\n#> 7 <split [3375/374]> Fold07\n#> 8 <split [3376/373]> Fold08\n#> 9 <split [3376/373]> Fold09\n#> 10 <split [3376/373]> Fold10" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", + "title": "2 - Tuning Hyperparameters", + "section": "Previously - Feature engineering ", + "text": "Previously - Feature engineering \n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#prepare-your-data-for-modeling", - "title": "2 - Feature Engineering", - "section": "Prepare your data for modeling ", - "text": "Prepare your data for modeling \n\nThe recipes package is an extensible framework for pipeable sequences of preprocessing and feature engineering steps.\n\n\n\nStatistical parameters for the steps can be estimated from an initial data set and then applied to other data sets.\n\n\n\n\nThe resulting processed output can be used as inputs for statistical or machine learning models." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-parameters", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#a-first-recipe", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#a-first-recipe", - "title": "2 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train)\n\n\n\nThe recipe() function assigns columns to roles of “outcome” or “predictor” using the formula" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#a-first-recipe-1", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#a-first-recipe-1", - "title": "2 - Feature Engineering", - "section": "A first recipe ", - "text": "A first recipe \n\nsummary(hotel_rec)\n#> # A tibble: 27 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 lead_time <chr [2]> predictor original\n#> 2 stays_in_weekend_nights <chr [2]> predictor original\n#> 3 stays_in_week_nights <chr [2]> predictor original\n#> 4 adults <chr [2]> predictor original\n#> 5 children <chr [2]> predictor original\n#> 6 babies <chr [2]> predictor original\n#> 7 meal <chr [3]> predictor original\n#> 8 country <chr [3]> predictor original\n#> 9 market_segment <chr [3]> predictor original\n#> 10 distribution_channel <chr [3]> predictor original\n#> # ℹ 17 more rows\n\nThe type column contains information on the variables" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", + "title": "2 - Tuning Hyperparameters", + "section": "Tagging parameters for tuning ", + "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn", - "title": "2 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\nWhat do you think are in the type vectors for the lead_time and country columns?\n\n\n\n−+\n02:00" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", + "title": "2 - Tuning Hyperparameters", + "section": "Optimizing the hash features ", + "text": "Optimizing the hash features \nOur new recipe is:\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\n\nWe will be using a tree-based model in a minute.\n\nThe other categorical predictors are left as-is.\nThat’s why there is no step_dummy()." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#create-indicator-variables", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#create-indicator-variables", - "title": "2 - Feature Engineering", - "section": "Create indicator variables ", - "text": "Create indicator variables \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors())\n\n\n\nFor any factor or character predictors, make binary indicators.\nThere are many recipe steps that can convert categorical predictors to numeric columns.\nstep_dummy() records the levels of the categorical predictors in the training set." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-trees", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-trees", + "title": "2 - Tuning Hyperparameters", + "section": "Boosted Trees", + "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved and new samples are predicted using a weighted average of the votes of each tree in the ensemble.\n\nWe’ll focus on the popular lightgbm implementation." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#filter-out-constant-columns", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#filter-out-constant-columns", - "title": "2 - Feature Engineering", - "section": "Filter out constant columns ", - "text": "Filter out constant columns \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors())\n\n\nIn case there is a factor level that was never observed in the training data (resulting in a column of all 0s), we can delete any zero-variance predictors that have a single unique value.\n\nNote that the selector chooses all columns with a role of “predictor”" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands)\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#normalization", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#normalization", - "title": "2 - Feature Engineering", - "section": "Normalization ", - "text": "Normalization \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors())\n\n\n\nThis centers and scales the numeric predictors.\nThe recipe will use the training set to estimate the means and standard deviations of the data.\n\n\n\n\nAll data the recipe is applied to will be normalized using those statistics (there is no re-estimation)." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", + "title": "2 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nTBH it is usually not difficult to optimize these models.\n\nOften, there are multiple candidate tuning parameter combinations that have very good results.\n\nTo demonstrate simple concepts, we’ll look at optimizing the number of trees in the ensemble (between 1 and 100) and the learning rate (\\(10^{-5}\\) to \\(10^{-1}\\))." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#reduce-correlation", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#reduce-correlation", - "title": "2 - Feature Engineering", - "section": "Reduce correlation ", - "text": "Reduce correlation \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n step_corr(all_numeric_predictors(), threshold = 0.9)\n\n\nTo deal with highly correlated predictors, find the minimum set of predictor columns that make the pairwise correlations less than the threshold." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", + "title": "2 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters ", + "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps", - "title": "2 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n step_pca(all_numeric_predictors())\n\n\nPCA feature extraction…" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "title": "2 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps-1", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps-1", - "title": "2 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n embed::step_umap(all_numeric_predictors(), outcome = vars(avg_price_per_room))\n\n\nA fancy machine learning supervised dimension reduction technique called UMAP…\n\nNote that this uses the outcome, and it is from an extension package" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search", + "title": "2 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps-2", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#other-possible-steps-2", - "title": "2 - Feature Engineering", - "section": "Other possible steps ", - "text": "Other possible steps \n\nhotel_rec <- \n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_normalize(all_numeric_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)\n\n\nNonlinear transforms like natural splines, and so on!" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-1", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-1", + "title": "2 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nIn reality we would probably sample the space more densely:" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn-1", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn-1", - "title": "2 - Feature Engineering", - "section": "Your turn", - "text": "Your turn\n\nCreate a recipe() for the hotel data to:\n\nuse a Yeo-Johnson (YJ) transformation on lead_time\nconvert factors to indicator variables\nremove zero-variance variables\nadd the spline technique shown above\n\n\n\n\n−+\n03:00" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#iterative-search", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#iterative-search", + "title": "2 - Tuning Hyperparameters", + "section": "Iterative Search", + "text": "Iterative Search\nWe could start with a few points and search the space:" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#minimal-recipe", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#minimal-recipe", - "title": "2 - Feature Engineering", - "section": "Minimal recipe ", - "text": "Minimal recipe \n\nhotel_indicators <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_YeoJohnson(lead_time) |>\n step_dummy(all_nominal_predictors()) |>\n step_zv(all_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#parameters", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Parameters", + "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#measuring-performance", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#measuring-performance", - "title": "2 - Feature Engineering", - "section": "Measuring Performance ", - "text": "Measuring Performance \nWe’ll compute two measures: mean absolute error and the coefficient of determination (a.k.a \\(R^2\\)).\n\\[\\begin{align}\nMAE &= \\frac{1}{n}\\sum_{i=1}^n |y_i - \\hat{y}_i| \\notag \\\\\nR^2 &= cor(y_i, \\hat{y}_i)^2\n\\end{align}\\]\nThe focus will be on MAE for parameter optimization. We’ll use a metric set to compute these:\n\nreg_metrics <- metric_set(mae, rsq)" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid", + "title": "2 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\nlgbm_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 4 parameters for tuning\n#> \n#> identifier type object\n#> trees trees nparam[+]\n#> learn_rate learn_rate nparam[+]\n#> agent hash num_terms nparam[+]\n#> company hash num_terms nparam[+]\n\n# Individual functions: \ntrees()\n#> # Trees (quantitative)\n#> Range: [1, 2000]\nlearn_rate()\n#> Learning Rate (quantitative)\n#> Transformer: log-10 [1e-100, Inf]\n#> Range (transformed scale): [-10, -1]\n\n\nA parameter set can be updated (e.g. to change the ranges)." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#using-a-workflow", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#using-a-workflow", - "title": "2 - Feature Engineering", - "section": "Using a workflow ", - "text": "Using a workflow \n\nset.seed(9)\n\nhotel_lm_wflow <-\n workflow() |>\n add_recipe(hotel_indicators) |>\n add_model(linear_reg())\n \nctrl <- control_resamples(save_pred = TRUE)\nhotel_lm_res <-\n hotel_lm_wflow |>\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_lm_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.6 10 0.214 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00339 Preprocessor1_Model1" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid-1", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid-1", + "title": "2 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\n\n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1629 0.00000440 524 1454\n#> 2 1746 0.0000000751 1009 2865\n#> 3 53 0.0000180 2313 367\n#> 4 442 0.000000445 347 460\n#> 5 1413 0.0000000208 3232 553\n#> 6 1488 0.0000578 3692 639\n#> 7 906 0.000385 602 332\n#> 8 1884 0.00000000101 1127 567\n#> 9 1812 0.0239 961 1183\n#> 10 393 0.000000117 487 1783\n#> # ℹ 15 more rows\n\n\n\n\nA space-filling design tends to perform better than random grids.\nSpace-filling designs are also usually more efficient than regular grids." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn-2", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#your-turn-2", - "title": "2 - Feature Engineering", + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn", + "title": "2 - Tuning Hyperparameters", "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() to fit your workflow with a recipe.\nCollect the predictions from the results.\n\n\n\n−+\n05:00" + "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#holdout-predictions", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#holdout-predictions", - "title": "2 - Feature Engineering", - "section": "Holdout predictions ", - "text": "Holdout predictions \n\n# Since we used `save_pred = TRUE`\nlm_cv_pred <- collect_predictions(hotel_lm_res)\nlm_cv_pred |> print(n = 7)\n#> # A tibble: 3,749 × 5\n#> .pred id .row avg_price_per_room .config \n#> <dbl> <chr> <int> <dbl> <chr> \n#> 1 75.1 Fold01 20 40 Preprocessor1_Model1\n#> 2 49.3 Fold01 28 54 Preprocessor1_Model1\n#> 3 64.9 Fold01 45 50 Preprocessor1_Model1\n#> 4 52.8 Fold01 49 42 Preprocessor1_Model1\n#> 5 48.6 Fold01 61 49 Preprocessor1_Model1\n#> 6 29.8 Fold01 66 40 Preprocessor1_Model1\n#> 7 36.9 Fold01 88 49 Preprocessor1_Model1\n#> # ℹ 3,742 more rows" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", + "title": "2 - Tuning Hyperparameters", + "section": "Create a regular grid ", + "text": "Create a regular grid \n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 4)\n\ngrid\n#> # A tibble: 256 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.0000000001 256 256\n#> 2 667 0.0000000001 256 256\n#> 3 1333 0.0000000001 256 256\n#> 4 2000 0.0000000001 256 256\n#> 5 1 0.0000001 256 256\n#> 6 667 0.0000001 256 256\n#> 7 1333 0.0000001 256 256\n#> 8 2000 0.0000001 256 256\n#> 9 1 0.0001 256 256\n#> 10 667 0.0001 256 256\n#> # ℹ 246 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#calibration-plot", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#calibration-plot", - "title": "2 - Feature Engineering", - "section": "Calibration Plot ", - "text": "Calibration Plot \n\nlibrary(probably)\n\ncal_plot_regression(hotel_lm_res)" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-1", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-1", + "title": "2 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\n\nWhat advantage would a regular grid have?" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#what-do-we-do-with-the-agent-and-company-data", - "title": "2 - Feature Engineering", - "section": "What do we do with the agent and company data?", - "text": "What do we do with the agent and company data?\nThere are 98 unique agent values and 100 unique companies in our training set. How can we include this information in our model?\n\nWe could:\n\nmake the full set of indicator variables 😳\nlump agents and companies that rarely occur into an “other” group\nuse feature hashing to create a smaller set of indicator variables\nuse effect encoding to replace the agent and company columns with the estimated effect of that predictor (in the extra materials)" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", + "title": "2 - Tuning Hyperparameters", + "section": "Update parameter ranges ", + "text": "Update parameter ranges \n\nlgbm_param <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n update(trees = trees(c(1L, 100L)),\n learn_rate = learn_rate(c(-5, -1)))\n\nset.seed(712)\ngrid <- \n lgbm_param %>% \n grid_latin_hypercube(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 75 0.000312 2991 1250\n#> 2 4 0.0000337 899 3088\n#> 3 15 0.0295 520 1578\n#> 4 8 0.0997 1256 3592\n#> 5 80 0.000622 419 258\n#> 6 70 0.000474 2499 1089\n#> 7 35 0.000165 287 2376\n#> 8 64 0.00137 389 359\n#> 9 58 0.0000250 616 881\n#> 10 84 0.0639 2311 2635\n#> # ℹ 15 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#per-agent-statistics", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#per-agent-statistics", - "title": "2 - Feature Engineering", - "section": "Per-agent statistics", - "text": "Per-agent statistics" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#the-results", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#the-results", + "title": "2 - Tuning Hyperparameters", + "section": "The results ", + "text": "The results \n\n\ngrid %>% \n ggplot(aes(trees, learn_rate)) +\n geom_point(size = 4) +\n scale_y_log10()\n\n\n\n\n\n\n\n\n\nNote that the learning rates are uniform on the log-10 scale." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#collapsing-factor-levels", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#collapsing-factor-levels", - "title": "2 - Feature Engineering", - "section": "Collapsing factor levels ", - "text": "Collapsing factor levels \nThere is a recipe step that will redefine factor levels based on their frequency in the training set:\n\nhotel_other_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |> \n step_YeoJohnson(lead_time) |>\n step_other(agent, threshold = 0.001) |>\n step_dummy(all_nominal_predictors()) |>\n step_zv(all_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nUsing this code, 34 agents (out of 98) were collapsed into “other” based on the training set.\nWe could try to optimize the threshold for collapsing (see the next set of slides on model tuning)." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", + "title": "2 - Tuning Hyperparameters", + "section": "Choosing tuning parameters ", + "text": "Choosing tuning parameters \nLet’s take our previous model and tune more parameters:\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)\n\n# Update the feature hash ranges (log-2 units)\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#does-othering-help", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#does-othering-help", - "title": "2 - Feature Engineering", - "section": "Does othering help? ", - "text": "Does othering help? \n\nhotel_other_wflow <-\n hotel_lm_wflow |>\n update_recipe(hotel_other_rec)\n\nhotel_other_res <-\n hotel_other_wflow |>\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_other_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.213 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00341 Preprocessor1_Model1\n\nAbout the same MAE and much faster to complete.\nNow let’s look at a more sophisticated tool called effect feature hashing." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-3", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-3", + "title": "2 - Tuning Hyperparameters", + "section": "Grid Search ", + "text": "Grid Search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE)\n\nlgbm_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = 25,\n # The options below are not required by default\n param_info = lgbm_param, \n control = ctrl,\n metrics = reg_metrics\n )\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing", - "title": "2 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nBetween agent and company, simple dummy variables would create 198 new columns (that are mostly zeros).\nAnother option is to have a binary indicator that combines some levels of these variables.\nFeature hashing (for more see FES, SMLTAR, TMwR, and aml4td):\n\nuses the character values of the levels\nconverts them to integer hash values\nuses the integers to assign them to a specific indicator column." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-4", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-4", + "title": "2 - Tuning Hyperparameters", + "section": "Grid Search ", + "text": "Grid Search \n\nlgbm_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [3372/377]> Fold01 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,425 × 9]>\n#> 2 <split [3373/376]> Fold02 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 3 <split [3373/376]> Fold03 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 4 <split [3373/376]> Fold04 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 5 <split [3373/376]> Fold05 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,400 × 9]>\n#> 6 <split [3374/375]> Fold06 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,375 × 9]>\n#> 7 <split [3375/374]> Fold07 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,350 × 9]>\n#> 8 <split [3376/373]> Fold08 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>\n#> 9 <split [3376/373]> Fold09 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>\n#> 10 <split [3376/373]> Fold10 <tibble [50 × 9]> <tibble [0 × 3]> <tibble [9,325 × 9]>" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-1", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-1", - "title": "2 - Feature Engineering", - "section": "Feature Hashing", - "text": "Feature Hashing\nSuppose we want to use 32 indicator variables for agent.\nFor a agent with value “Max_Kuhn”, a hashing function converts it to an integer (say 210397726).\nTo assign it to one of the 32 columns, we would use modular arithmetic to assign it to a column:\n\n# For \"Max_Kuhn\" put a '1' in column: \n210397726 %% 32\n#> [1] 30\n\nHash functions are meant to emulate randomness." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-results", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#grid-results", + "title": "2 - Tuning Hyperparameters", + "section": "Grid results ", + "text": "Grid results \n\nautoplot(lgbm_res)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-pros", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-pros", - "title": "2 - Feature Engineering", - "section": "Feature Hashing Pros", - "text": "Feature Hashing Pros\n\nThe procedure will automatically work on new values of the predictors.\nIt is fast.\n“Signed” hashes add a sign to help avoid aliasing." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results", + "title": "2 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(lgbm_res)\n#> # A tibble: 50 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 298 19 4.15e- 9 222 36 mae standard 53.2 10 0.427 Preprocessor01_Model1\n#> 2 298 19 4.15e- 9 222 36 rsq standard 0.810 10 0.00686 Preprocessor01_Model1\n#> 3 1394 5 5.82e- 6 28 21 mae standard 52.9 10 0.424 Preprocessor02_Model1\n#> 4 1394 5 5.82e- 6 28 21 rsq standard 0.810 10 0.00800 Preprocessor02_Model1\n#> 5 774 12 4.41e- 2 27 95 mae standard 9.77 10 0.155 Preprocessor03_Model1\n#> 6 774 12 4.41e- 2 27 95 rsq standard 0.946 10 0.00341 Preprocessor03_Model1\n#> 7 1342 7 6.84e-10 71 17 mae standard 53.2 10 0.427 Preprocessor04_Model1\n#> 8 1342 7 6.84e-10 71 17 rsq standard 0.811 10 0.00785 Preprocessor04_Model1\n#> 9 669 39 8.62e- 7 141 145 mae standard 53.2 10 0.426 Preprocessor05_Model1\n#> 10 669 39 8.62e- 7 141 145 rsq standard 0.807 10 0.00639 Preprocessor05_Model1\n#> # ℹ 40 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-cons", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-cons", - "title": "2 - Feature Engineering", - "section": "Feature Hashing Cons", - "text": "Feature Hashing Cons\n\nThere is no real logic behind which factor levels are combined.\nWe don’t know how many columns to add (more in the next section).\nSome columns may have all zeros.\nIf a indicator column is important to the model, we can’t easily determine why.\n\n\nThe signed hash make it slightly more possible to differentiate between confounded levels" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results-1", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results-1", + "title": "2 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(lgbm_res, summarize = FALSE)\n#> # A tibble: 500 × 10\n#> id trees min_n learn_rate `agent hash` `company hash` .metric .estimator .estimate .config \n#> <chr> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 Fold01 298 19 0.00000000415 222 36 mae standard 51.8 Preprocessor01_Model1\n#> 2 Fold01 298 19 0.00000000415 222 36 rsq standard 0.821 Preprocessor01_Model1\n#> 3 Fold02 298 19 0.00000000415 222 36 mae standard 52.1 Preprocessor01_Model1\n#> 4 Fold02 298 19 0.00000000415 222 36 rsq standard 0.804 Preprocessor01_Model1\n#> 5 Fold03 298 19 0.00000000415 222 36 mae standard 52.2 Preprocessor01_Model1\n#> 6 Fold03 298 19 0.00000000415 222 36 rsq standard 0.786 Preprocessor01_Model1\n#> 7 Fold04 298 19 0.00000000415 222 36 mae standard 51.7 Preprocessor01_Model1\n#> 8 Fold04 298 19 0.00000000415 222 36 rsq standard 0.826 Preprocessor01_Model1\n#> 9 Fold05 298 19 0.00000000415 222 36 mae standard 55.2 Preprocessor01_Model1\n#> 10 Fold05 298 19 0.00000000415 222 36 rsq standard 0.845 Preprocessor01_Model1\n#> # ℹ 490 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes", - "title": "2 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \nThe textrecipes package has a step that can be added to the recipe:\n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) |>\n step_dummy_hash(company) |>\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors()) |> \n step_spline_natural(arrival_date_num, deg_free = 10)\n\nhotel_hash_wflow <-\n hotel_lm_wflow |>\n update_recipe(hash_rec)" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", + "title": "2 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \n\nshow_best(lgbm_res, metric = \"rsq\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1890 10 0.0159 115 174 rsq standard 0.948 10 0.00334 Preprocessor12_Model1\n#> 2 774 12 0.0441 27 95 rsq standard 0.946 10 0.00341 Preprocessor03_Model1\n#> 3 1638 36 0.0409 15 120 rsq standard 0.945 10 0.00384 Preprocessor16_Model1\n#> 4 963 23 0.00556 157 13 rsq standard 0.937 10 0.00320 Preprocessor06_Model1\n#> 5 590 5 0.00320 85 73 rsq standard 0.908 10 0.00465 Preprocessor24_Model1" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#feature-hashing-in-recipes-1", - "title": "2 - Feature Engineering", - "section": "Feature Hashing in recipes ", - "text": "Feature Hashing in recipes \n\nhotel_hash_res <-\n hotel_hash_wflow |>\n fit_resamples(hotel_rs, control = ctrl, metrics = reg_metrics)\n\ncollect_metrics(hotel_hash_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 mae standard 16.7 10 0.239 Preprocessor1_Model1\n#> 2 rsq standard 0.884 10 0.00324 Preprocessor1_Model1\n\nAbout the same performance but now we can handle new values." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "title": "2 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nlgbm_best <- select_best(lgbm_res, metric = \"mae\")\nlgbm_best\n#> # A tibble: 1 × 6\n#> trees min_n learn_rate `agent hash` `company hash` .config \n#> <int> <int> <dbl> <int> <int> <chr> \n#> 1 1890 10 0.0159 115 174 Preprocessor12_Model1" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#debugging-a-recipe", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#debugging-a-recipe", - "title": "2 - Feature Engineering", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. hash_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4\n\n\n\n\nAnother function (bake()) is analogous to predict(), and gives you the processed data back.\n\n\n\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#checking-calibration", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#checking-calibration", + "title": "2 - Tuning Hyperparameters", + "section": "Checking Calibration ", + "text": "Checking Calibration \n\n\nlibrary(probably)\nlgbm_res %>%\n collect_predictions(\n parameters = lgbm_best\n ) %>%\n cal_plot_regression(\n truth = avg_price_per_room,\n estimate = .pred,\n alpha = 1 / 3\n )" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#example", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#example", - "title": "2 - Feature Engineering", - "section": "Example ", - "text": "Example \n\nhash_rec_fit <- prep(hash_rec)\n\n# Get the transformation coefficient\ntidy(hash_rec_fit, number = 1)\n\n# Get the processed data\nbake(hash_rec_fit, hotel_train |> slice(1:3), contains(\"_agent_\"))" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel", + "title": "2 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#more-on-recipes", - "href": "archive/2025-08-nyr/slides/advanced-02-feature-engineering.html#more-on-recipes", - "title": "2 - Feature Engineering", - "section": "More on recipes", - "text": "More on recipes\n\nOnce fit() is called on a workflow, changing the model does not re-fit the recipe.\n\n\n\nA list of all known steps is at https://www.tidymodels.org/find/recipes/.\n\n\n\n\nSome steps can be skipped when using predict().\n\n\n\n\nThe order of the steps matters." + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", + "title": "2 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#who-are-you", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML\nYou have used some tidymodels packages\nYou have some experience with evaluating statistical models using resampling techniques" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", + "title": "2 - Tuning Hyperparameters", + "section": "Early stopping for boosted trees", + "text": "Early stopping for boosted trees\nWe have directly optimized the number of trees as a tuning parameter.\nInstead we could\n\nSet the number of trees to a single large number.\nStop adding trees when performance gets worse.\n\nThis is known as “early stopping” and there is a parameter for that: stop_iter.\nEarly stopping has a potential to decrease the tuning time." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#who-are-tidymodels", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#who-are-tidymodels", - "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-2", + "href": "archive/2023-09-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-2", + "title": "2 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\n\nSet trees = 2000 and tune the stop_iter parameter.\nNote that you will need to regenerate lgbm_param with your new workflow!\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-2", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-2", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---setup", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---setup", + "title": "4 - Iterative Search", + "section": "Previously - Setup", + "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-3", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-3", - "title": "1 - Introduction", - "section": "👀", - "text": "👀\n\n\nhttps://workshops.tidymodels.org" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---data-usage", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---data-usage", + "title": "4 - Iterative Search", + "section": "Previously - Data Usage", + "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#tentative-plan-for-this-workshop", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#tentative-plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Tentative plan for this workshop", - "text": "Tentative plan for this workshop\n\nFeature engineering with recipes\nModel optimization by tuning\n\nGrid search\nRacing\nIterative methods\n\nExtras (time permitting)\n\nEffect encodings\nA case study" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---boosting-model", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#previously---boosting-model", + "title": "4 - Iterative Search", + "section": "Previously - Boosting Model", + "text": "Previously - Boosting Model\n\nhotel_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hotel_rec, lgbm_spec)\n\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-4", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#section-4", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#iterative-search", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#iterative-search", + "title": "4 - Iterative Search", + "section": "Iterative Search", + "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our boosted tree.\n\nWe could do something like:\nmae_pred <- lm(mae ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#lets-install-some-packages", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"lightgbm\", \"lme4\", \"pak\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\npak::pak(\"simonpcouch/forested\")\n\nAlso, you should install the newest version of the dials package (version 1.3.0). To check this, you can run:\n\nrlang::check_installed(\"dials\", version = \"1.3.0\")" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#iterative-search-1", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#iterative-search-1", + "title": "4 - Iterative Search", + "section": "Iterative Search", + "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between MAE and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#hotel-data", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#hotel-data", - "title": "1 - Introduction", - "section": "Hotel Data ", - "text": "Hotel Data \nWe’ll use data on hotels to predict the cost of a room.\nThe data are in the modeldata package. We’ll sample down the data and refactor some columns:\n\n\n\nlibrary(tidymodels)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#a-large-grid", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#a-large-grid", + "title": "4 - Iterative Search", + "section": "A Large Grid", + "text": "A Large Grid" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#hotel-date-columns", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#hotel-date-columns", - "title": "1 - Introduction", - "section": "Hotel date columns", - "text": "Hotel date columns\n\nnames(hotel_rates)\n#> [1] \"avg_price_per_room\" \"lead_time\" \n#> [3] \"stays_in_weekend_nights\" \"stays_in_week_nights\" \n#> [5] \"adults\" \"children\" \n#> [7] \"babies\" \"meal\" \n#> [9] \"country\" \"market_segment\" \n#> [11] \"distribution_channel\" \"is_repeated_guest\" \n#> [13] \"previous_cancellations\" \"previous_bookings_not_canceled\"\n#> [15] \"reserved_room_type\" \"assigned_room_type\" \n#> [17] \"booking_changes\" \"agent\" \n#> [19] \"company\" \"days_in_waiting_list\" \n#> [21] \"customer_type\" \"required_car_parking_spaces\" \n#> [23] \"total_of_special_requests\" \"arrival_date_num\" \n#> [25] \"near_christmas\" \"near_new_years\" \n#> [27] \"historical_adr\"" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#a-three-point-grid", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#a-three-point-grid", + "title": "4 - Iterative Search", + "section": "A Three Point Grid", + "text": "A Three Point Grid" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#data-splitting-strategy", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#data-splitting-strategy", - "title": "1 - Introduction", - "section": "Data splitting strategy", - "text": "Data splitting strategy" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#gaussian-processes-and-optimization", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#gaussian-processes-and-optimization", + "title": "4 - Iterative Search", + "section": "Gaussian Processes and Optimization", + "text": "Gaussian Processes and Optimization\nWe can make a “meta-model” with a small set of historical performance results.\nGaussian Processes (GP) models are a good choice to model performance.\n\nIt is a Bayesian model so we are using Bayesian Optimization (BO).\nFor regression, we can assume that our data are multivariate normal.\nWe also define a covariance function for the variance relationship between data points. A common one is:\n\n\\[\\operatorname{cov}(\\boldsymbol{x}_i, \\boldsymbol{x}_j) = \\exp\\left(-\\frac{1}{2}|\\boldsymbol{x}_i - \\boldsymbol{x}_j|^2\\right) + \\sigma^2_{ij}\\]\n\nGPs are good because\n\nthey are flexible regression models (in the sense that splines are flexible).\nwe need to get mean and variance predictions (and they are Bayesian)\ntheir variability is based on spatial distances.\n\nSome people use random forests (with conformal variance estimates) or other methods but GPs are most popular." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#data-spending", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#data-spending", - "title": "1 - Introduction", - "section": "Data Spending ", - "text": "Data Spending \nLet’s split the data into a training set (75%) and testing set (25%) using stratification:\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#predicting-candidates", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#predicting-candidates", + "title": "4 - Iterative Search", + "section": "Predicting Candidates", + "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. MAE)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#your-turn", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#your-turn", - "title": "1 - Introduction", + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#your-turn", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#your-turn", + "title": "4 - Iterative Search", "section": "Your turn", - "text": "Your turn\nLet’s take some time and investigate the training data. The outcome is avg_price_per_room.\nAre there any interesting characteristics of the data?\n\n\n\n−+\n10:00" + "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize MAE.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-01-introduction.html#our-versions", - "href": "archive/2025-08-nyr/slides/advanced-01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nR version 4.5.0 (2025-04-11), Quarto (1.6.42)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.4.0\n\n\nbroom\n1.0.9\n\n\nCubist\n0.5.0\n\n\ndials\n1.4.1\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.4\n\n\nembed\n1.1.5.9000\n\n\nfinetune\n1.2.1.9000\n\n\nforested\n0.2.0\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nFormula\n1.2-5\n\n\nggplot2\n3.5.2\n\n\nlattice\n0.22-7\n\n\nlightgbm\n4.6.0\n\n\nlme4\n1.1-37\n\n\nmodeldata\n1.5.1\n\n\nparallelly\n1.45.1\n\n\nparsnip\n1.3.2.9000\n\n\nplotmo\n3.6.4\n\n\nplotrix\n3.8-4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nplumber\n1.3.0\n\n\nprobably\n1.1.0\n\n\npurrr\n1.1.0\n\n\nrecipes\n1.3.1\n\n\nrsample\n1.3.1\n\n\nrules\n1.0.2\n\n\nscales\n1.4.0\n\n\nsplines2\n0.5.4\n\n\nstacks\n1.1.1.9000\n\n\ntext2vec\n0.6.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntextrecipes\n1.1.0.9000\n\n\ntibble\n3.3.0\n\n\ntidymodels\n1.3.0\n\n\ntidyr\n1.3.1\n\n\ntune\n1.3.0\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.2.0.9002\n\n\nworkflowsets\n1.1.1\n\n\nyardstick\n1.3.2" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", + "title": "4 - Iterative Search", + "section": "GP Fit (ribbon is mean +/- 1SD)", + "text": "GP Fit (ribbon is mean +/- 1SD)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---setup", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---setup", - "title": "3 - Tuning Hyperparameters", - "section": "Previously - Setup ", - "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#choosing-new-candidates", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#choosing-new-candidates", + "title": "4 - Iterative Search", + "section": "Choosing New Candidates", + "text": "Choosing New Candidates\nThis isn’t a very good fit but we can still use it.\nHow can we use the outputs to choose the next point to measure?\n\nAcquisition functions take the predicted mean and variance and use them to balance:\n\nexploration: new candidates should explore new areas.\nexploitation: new candidates must stay near existing values.\n\nExploration focuses on the variance, exploitation is about the mean." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---data-usage", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---data-usage", - "title": "3 - Tuning Hyperparameters", - "section": "Previously - Data Usage ", - "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#acquisition-functions", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#acquisition-functions", + "title": "4 - Iterative Search", + "section": "Acquisition Functions", + "text": "Acquisition Functions\nWe’ll use an acquisition function to select a new candidate.\nThe most popular method appears to be expected improvement (EI) above the current best results.\n\nZero at existing data points.\nThe expected improvement is integrated over all possible improvement (“expected” in the probability sense).\n\nWe would probably pick the point with the largest EI as the next point.\n(There are other functions beyond EI.)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", - "title": "3 - Tuning Hyperparameters", - "section": "Previously - Feature engineering ", - "text": "Previously - Feature engineering \n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) |>\n step_dummy_hash(company) |>\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) |> \n step_zv(all_predictors())" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement", + "title": "4 - Iterative Search", + "section": "Expected Improvement", + "text": "Expected Improvement" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-parameters", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#iteration", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#iteration", + "title": "4 - Iterative Search", + "section": "Iteration", + "text": "Iteration\nOnce we pick the candidate point, we measure performance for it (e.g. resampling).\n\nAnother GP is fit, EI is recomputed, and so on.\n\nWe stop when we have completed the allowed number of iterations or if we don’t see any improvement after a pre-set number of attempts." }, - { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + { + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-fit-with-four-points", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-fit-with-four-points", + "title": "4 - Iterative Search", + "section": "GP Fit with four points", + "text": "GP Fit with four points" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", - "title": "3 - Tuning Hyperparameters", - "section": "Tagging parameters for tuning ", - "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement-1", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement-1", + "title": "4 - Iterative Search", + "section": "Expected Improvement", + "text": "Expected Improvement" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", - "title": "3 - Tuning Hyperparameters", - "section": "Optimizing the hash features ", - "text": "Optimizing the hash features \nOur new recipe is:\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_YeoJohnson(lead_time) |>\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) |>\n step_dummy_hash(company, num_terms = tune(\"company hash\")) |>\n step_zv(all_predictors())\n\n\nWe will be using a tree-based model in a minute.\n\nThe other categorical predictors are left as-is.\nThat’s why there is no step_dummy()." + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-evolution", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#gp-evolution", + "title": "4 - Iterative Search", + "section": "GP Evolution", + "text": "GP Evolution" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-trees", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-trees", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Trees", - "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved and new samples are predicted using a weighted average of the votes of each tree in the ensemble.\n\nWe’ll focus on the popular lightgbm implementation." + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement-evolution", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#expected-improvement-evolution", + "title": "4 - Iterative Search", + "section": "Expected Improvement Evolution", + "text": "Expected Improvement Evolution" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands)\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#bo-in-tidymodels", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#bo-in-tidymodels", + "title": "4 - Iterative Search", + "section": "BO in tidymodels", + "text": "BO in tidymodels\nWe’ll use a function called tune_bayes() that has very similar syntax to tune_grid().\n\nIt has an additional initial argument for the initial set of performance estimates and parameter combinations for the GP model." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters", - "text": "Boosted Tree Tuning Parameters\nTBH it is usually not difficult to optimize these models.\n\nOften, there are multiple candidate tuning parameter combinations that have very good results.\n\nTo demonstrate simple concepts, we’ll look at optimizing the number of trees in the ensemble (between 1 and 100) and the learning rate (\\(10^{-5}\\) to \\(10^{-1}\\))." + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#initial-grid-points", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#initial-grid-points", + "title": "4 - Iterative Search", + "section": "Initial grid points", + "text": "Initial grid points\ninitial can be the results of another tune_*() function or an integer (in which case tune_grid() is used under to hood to make such an initial set of results).\n\nWe’ll run the optimization more than once, so let’s make an initial grid of results to serve as the substrate for the BO.\nI suggest at least the number of tuning parameters plus two as the initial grid for BO." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", - "title": "3 - Tuning Hyperparameters", - "section": "Boosted Tree Tuning Parameters ", - "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#an-initial-grid", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#an-initial-grid", + "title": "4 - Iterative Search", + "section": "An Initial Grid", + "text": "An Initial Grid\n\nreg_metrics <- metric_set(mae, rsq)\n\nset.seed(12)\ninit_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = nrow(lgbm_param) + 2,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(init_res, metric = \"mae\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 390 10 0.0139 13 62 mae standard 11.3 10 0.202 Preprocessor1_Model1\n#> 2 718 31 0.00112 72 25 mae standard 29.0 10 0.335 Preprocessor4_Model1\n#> 3 1236 22 0.0000261 11 17 mae standard 51.8 10 0.416 Preprocessor7_Model1\n#> 4 1044 25 0.00000832 34 12 mae standard 52.8 10 0.424 Preprocessor5_Model1\n#> 5 1599 7 0.0000000402 254 179 mae standard 53.2 10 0.427 Preprocessor6_Model1" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", - "title": "3 - Tuning Hyperparameters", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#bo-using-tidymodels", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#bo-using-tidymodels", + "title": "4 - Iterative Search", + "section": "BO using tidymodels", + "text": "BO using tidymodels\n\nset.seed(15)\nlgbm_bayes_res <-\n lgbm_wflow %>%\n tune_bayes(\n resamples = hotel_rs,\n initial = init_res, # <- initial results\n iter = 20,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(lgbm_bayes_res, metric = \"mae\")\n#> # A tibble: 5 × 12\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config .iter\n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> <int>\n#> 1 1769 2 0.0299 114 245 mae standard 9.41 10 0.160 Iter13 13\n#> 2 1969 3 0.0270 240 99 mae standard 9.49 10 0.189 Iter11 11\n#> 3 1780 5 0.0403 27 78 mae standard 9.54 10 0.164 Iter17 17\n#> 4 1454 3 0.0414 114 10 mae standard 9.55 10 0.144 Iter10 10\n#> 5 1253 2 0.0312 131 207 mae standard 9.56 10 0.145 Iter19 19" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search", - "title": "3 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results", + "title": "4 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\")" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-1", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-1", - "title": "3 - Tuning Hyperparameters", - "section": "Grid search", - "text": "Grid search\nIn reality we would probably sample the space more densely:" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results-1", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results-1", + "title": "4 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"parameters\")" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#iterative-search", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#iterative-search", - "title": "3 - Tuning Hyperparameters", - "section": "Iterative Search", - "text": "Iterative Search\nWe could start with a few points and search the space:" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results-2", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#plotting-bo-results-2", + "title": "4 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\")" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#parameters", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Parameters", - "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#enhance", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#enhance", + "title": "4 - Iterative Search", + "section": "ENHANCE", + "text": "ENHANCE\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\") +\n ylim(c(9, 14))" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#different-types-of-grids", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#different-types-of-grids", - "title": "3 - Tuning Hyperparameters", - "section": "Different types of grids ", - "text": "Different types of grids \n\n\n\n\n\n\n\n\n\nSpace-filling designs (SFD) attempt to cover the parameter space without redundant candidates. We recommend these the most." + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#your-turn-1", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#your-turn-1", + "title": "4 - Iterative Search", + "section": "Your turn", + "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\n\n\n\n−+\n10:00" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-grid", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-grid", - "title": "3 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\nlgbm_wflow |> \n extract_parameter_set_dials()\n\n# Individual functions: \ntrees()\nlearn_rate()\n\n\nA parameter set can be updated (e.g. to change the ranges)." + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#notes", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#notes", + "title": "4 - Iterative Search", + "section": "Notes", + "text": "Notes\n\nStopping tune_bayes() will return the current results.\nParallel processing can still be used to more efficiently measure each candidate point.\nThere are a lot of other iterative methods that you can use.\nThe finetune package also has functions for simulated annealing search." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-grid-1", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-grid-1", - "title": "3 - Tuning Hyperparameters", - "section": "Create a grid ", - "text": "Create a grid \n\ngrid <- \n lgbm_wflow |> \n extract_parameter_set_dials() |> \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 7.50e- 6 574 574\n#> 2 84 1.78e- 5 2048 2298\n#> 3 167 5.62e-10 1824 912\n#> 4 250 4.22e- 5 3250 512\n#> 5 334 1.78e- 8 512 2896\n#> 6 417 1.33e- 3 322 1625\n#> 7 500 1 e- 1 1448 1149\n#> 8 584 1 e- 7 1290 256\n#> 9 667 2.37e-10 456 724\n#> 10 750 1.78e- 2 645 322\n#> # ℹ 15 more rows" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#finalizing-the-model", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#finalizing-the-model", + "title": "4 - Iterative Search", + "section": "Finalizing the Model", + "text": "Finalizing the Model\nLet’s say that we’ve tried a lot of different models and we like our lightgbm model the most.\nWhat do we do now?\n\nFinalize the workflow by choosing the values for the tuning parameters.\nFit the model on the entire training set.\nVerify performance using the test set.\nDocument and publish the model(?)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn", - "title": "3 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n−+\n03:00" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#locking-down-the-tuning-parameters", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#locking-down-the-tuning-parameters", + "title": "4 - Iterative Search", + "section": "Locking Down the Tuning Parameters", + "text": "Locking Down the Tuning Parameters\nWe can take the results of the Bayesian optimization and accept the best results:\n\nbest_param <- select_best(lgbm_bayes_res, metric = \"mae\")\nfinal_wflow <- \n lgbm_wflow %>% \n finalize_workflow(best_param)\nfinal_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: boost_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 4 Recipe Steps\n#> \n#> • step_YeoJohnson()\n#> • step_dummy_hash()\n#> • step_dummy_hash()\n#> • step_zv()\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Boosted Tree Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1769\n#> min_n = 2\n#> learn_rate = 0.0299391312149257\n#> \n#> Engine-Specific Arguments:\n#> num_threads = 1\n#> \n#> Computational engine: lightgbm" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", - "title": "3 - Tuning Hyperparameters", - "section": "Create a regular grid ", - "text": "Create a regular grid \n\ngrid <- \n lgbm_wflow |> \n extract_parameter_set_dials() |> \n grid_regular(levels = 4)\n\ngrid\n#> # A tibble: 256 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.0000000001 256 256\n#> 2 667 0.0000000001 256 256\n#> 3 1333 0.0000000001 256 256\n#> 4 2000 0.0000000001 256 256\n#> 5 1 0.0000001 256 256\n#> 6 667 0.0000001 256 256\n#> 7 1333 0.0000001 256 256\n#> 8 2000 0.0000001 256 256\n#> 9 1 0.0001 256 256\n#> 10 667 0.0001 256 256\n#> # ℹ 246 more rows" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#the-final-fit", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#the-final-fit", + "title": "4 - Iterative Search", + "section": "The Final Fit", + "text": "The Final Fit\nWe can use individual functions:\nfinal_fit <- final_wflow %>% fit(data = hotel_train)\n\n# then predict() or augment() \n# then compute metrics\n\nRemember that there is also a convenience function to do all of this:\n\nset.seed(3893)\nfinal_res <- final_wflow %>% last_fit(hotel_split, metrics = reg_metrics)\nfinal_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [3749/1251]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [1,251 × 4]> <workflow>" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn-1", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn-1", - "title": "3 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\n\nWhat advantage would a regular grid have?" + "objectID": "archive/2023-09-posit-conf/advanced-05-iterative.html#test-set-results", + "href": "archive/2023-09-posit-conf/advanced-05-iterative.html#test-set-results", + "title": "4 - Iterative Search", + "section": "Test Set Results", + "text": "Test Set Results\n\n\n\nfinal_res %>% \n collect_predictions() %>% \n cal_plot_regression(\n truth = avg_price_per_room, \n estimate = .pred, \n alpha = 1 / 4)\n\nTest set performance:\n\nfinal_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 9.63 Preprocessor1_Model1\n#> 2 rsq standard 0.948 Preprocessor1_Model1" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", - "title": "3 - Tuning Hyperparameters", - "section": "Update parameter ranges ", - "text": "Update parameter ranges \n\nlgbm_param <- \n lgbm_wflow |> \n extract_parameter_set_dials() |> \n update(trees = trees(c(1L, 100L)),\n learn_rate = learn_rate(c(-5, -1)))\n\ngrid <- \n lgbm_param |> \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.00147 574 574\n#> 2 5 0.00215 2048 2298\n#> 3 9 0.0000215 1824 912\n#> 4 13 0.00316 3250 512\n#> 5 17 0.0001 512 2896\n#> 6 21 0.0147 322 1625\n#> 7 25 0.1 1448 1149\n#> 8 29 0.000215 1290 256\n#> 9 34 0.0000147 456 724\n#> 10 38 0.0464 645 322\n#> # ℹ 15 more rows" + "objectID": "archive/2023-09-posit-conf/annotations.html#section", + "href": "archive/2023-09-posit-conf/annotations.html#section", + "title": "Annotations", + "section": "👀", + "text": "👀\nThis page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#the-results", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#the-results", - "title": "3 - Tuning Hyperparameters", - "section": "The results ", - "text": "The results \n\n\ngrid |> \n ggplot(aes(trees, learn_rate)) +\n geom_point(size = 4) +\n scale_y_log10()\n\n\n\n\n\n\n\n\n\nNote that the learning rates are uniform on the log-10 scale and this shows 2 of 4 dimensions." + "objectID": "archive/2023-09-posit-conf/annotations.html#the-initial-split", + "href": "archive/2023-09-posit-conf/annotations.html#the-initial-split", + "title": "Annotations", + "section": "The initial split", + "text": "The initial split\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", - "title": "3 - Tuning Hyperparameters", - "section": "Choosing tuning parameters ", - "text": "Choosing tuning parameters \nLet’s take our previous model and tune more parameters:\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)\n\n# Update the feature hash ranges (log-2 units)\nlgbm_param <-\n lgbm_wflow |>\n extract_parameter_set_dials() |>\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" + "objectID": "archive/2023-09-posit-conf/annotations.html#what-is-wrong-with-this", + "href": "archive/2023-09-posit-conf/annotations.html#what-is-wrong-with-this", + "title": "Annotations", + "section": "What is wrong with this?", + "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-3", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-3", - "title": "3 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE)\n\nlgbm_res <-\n lgbm_wflow |>\n tune_grid(\n resamples = hotel_rs,\n grid = 25,\n # The options below are not required by default\n param_info = lgbm_param, \n control = ctrl,\n metrics = reg_metrics\n )\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" + "objectID": "archive/2023-09-posit-conf/annotations.html#where-are-the-fitted-models", + "href": "archive/2023-09-posit-conf/annotations.html#where-are-the-fitted-models", + "title": "Annotations", + "section": "Where are the fitted models?", + "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-4", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-search-4", - "title": "3 - Tuning Hyperparameters", - "section": "Grid Search ", - "text": "Grid Search \n\nlgbm_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [3372/377]> Fold01 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,425 × 9]>\n#> 2 <split [3373/376]> Fold02 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 3 <split [3373/376]> Fold03 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 4 <split [3373/376]> Fold04 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 5 <split [3373/376]> Fold05 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 6 <split [3374/375]> Fold06 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,375 × 9]>\n#> 7 <split [3375/374]> Fold07 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,350 × 9]>\n#> 8 <split [3376/373]> Fold08 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 9 <split [3376/373]> Fold09 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 10 <split [3376/373]> Fold10 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>" + "objectID": "archive/2023-09-posit-conf/annotations.html#update-parameter-ranges", + "href": "archive/2023-09-posit-conf/annotations.html#update-parameter-ranges", + "title": "Annotations", + "section": "Update parameter ranges", + "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_spline_natural(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-results", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#grid-results", - "title": "3 - Tuning Hyperparameters", - "section": "Grid results ", - "text": "Grid results \n\nautoplot(lgbm_res)" + "objectID": "archive/2023-09-posit-conf/annotations.html#early-stopping-for-boosted-trees", + "href": "archive/2023-09-posit-conf/annotations.html#early-stopping-for-boosted-trees", + "title": "Annotations", + "section": "Early stopping for boosted trees", + "text": "Early stopping for boosted trees\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the lightgbm package.\nbonsai has its own wrapper around (lightgbm::lgb.train()) called bonsai::train_lightgbm(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the lightgbm entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for lightgbm)." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-results", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-results", - "title": "3 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res)\n#> # A tibble: 50 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1500 21 0.00000000316 8 60 mae standard 53.2 10 0.427 pre01_mod19_post0\n#> 2 1500 21 0.00000000316 8 60 rsq standard 0.809 10 0.00661 pre01_mod19_post0\n#> 3 917 16 0.0422 9 39 mae standard 9.85 10 0.150 pre02_mod12_post0\n#> 4 917 16 0.0422 9 39 rsq standard 0.946 10 0.00362 pre02_mod12_post0\n#> 5 584 13 0.0000001 10 10 mae standard 53.2 10 0.427 pre03_mod08_post0\n#> 6 584 13 0.0000001 10 10 rsq standard 0.810 10 0.00745 pre03_mod08_post0\n#> 7 167 19 0.00000133 12 143 mae standard 53.2 10 0.426 pre04_mod03_post0\n#> 8 167 19 0.00000133 12 143 rsq standard 0.811 10 0.00698 pre04_mod03_post0\n#> 9 1583 33 0.0000422 14 12 mae standard 50.3 10 0.405 pre05_mod20_post0\n#> 10 1583 33 0.0000422 14 12 rsq standard 0.817 10 0.00768 pre05_mod20_post0\n#> # ℹ 40 more rows" + "objectID": "archive/2023-09-posit-conf/annotations.html#per-agent-statistics", + "href": "archive/2023-09-posit-conf/annotations.html#per-agent-statistics", + "title": "Annotations", + "section": "Per-agent statistics", + "text": "Per-agent statistics\nThe effect encoding method essentially takes the effect of a variable, like agent, and makes a data column for that effect. In our example, affect of the agent on the ADR is quantified by a model and then added as a data column to be used in the model.\nSuppose agent Max has a single reservation in the data and it had an ADR of €200. If we used a naive estimate for Max’s effect, the model is being told that Max should always produce an effect of €200. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned agent Davis, who has taken 250 reservations with an average ADR of €100. Davis’s mean is more predictive because it is estimated with better data (i.e., more total reservations). Partial pooling leverages the entire data set and can borrow strength from all of the agents. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a agent’s data is of good quality, the partial pooling effect estimate is closer to the raw mean. Max’s data is not great and is “shrunk” towards the center of the overall average. Since there is so little known about Max’s reservation history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nSince this example has a numeric outcome, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-results-1", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#tuning-results-1", - "title": "3 - Tuning Hyperparameters", - "section": "Tuning results ", - "text": "Tuning results \n\ncollect_metrics(lgbm_res, summarize = FALSE)\n#> # A tibble: 500 × 10\n#> id trees min_n learn_rate `agent hash` `company hash` .metric .estimator .estimate .config \n#> <chr> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 Fold01 1500 21 0.00000000316 8 60 mae standard 51.8 pre01_mod19_post0\n#> 2 Fold01 1500 21 0.00000000316 8 60 rsq standard 0.805 pre01_mod19_post0\n#> 3 Fold02 1500 21 0.00000000316 8 60 mae standard 52.1 pre01_mod19_post0\n#> 4 Fold02 1500 21 0.00000000316 8 60 rsq standard 0.800 pre01_mod19_post0\n#> 5 Fold03 1500 21 0.00000000316 8 60 mae standard 52.2 pre01_mod19_post0\n#> 6 Fold03 1500 21 0.00000000316 8 60 rsq standard 0.783 pre01_mod19_post0\n#> 7 Fold04 1500 21 0.00000000316 8 60 mae standard 51.7 pre01_mod19_post0\n#> 8 Fold04 1500 21 0.00000000316 8 60 rsq standard 0.818 pre01_mod19_post0\n#> 9 Fold05 1500 21 0.00000000316 8 60 mae standard 55.2 pre01_mod19_post0\n#> 10 Fold05 1500 21 0.00000000316 8 60 rsq standard 0.845 pre01_mod19_post0\n#> # ℹ 490 more rows" + "objectID": "archive/2023-09-posit-conf/annotations.html#agent-effects", + "href": "archive/2023-09-posit-conf/annotations.html#agent-effects", + "title": "Annotations", + "section": "Agent effects", + "text": "Agent effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage though." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", - "title": "3 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \n\nshow_best(lgbm_res, metric = \"rsq\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1250 9 0.1 107 80 rsq standard 0.947 10 0.00395 pre19_mod16_post0\n#> 2 917 16 0.0422 9 39 rsq standard 0.946 10 0.00362 pre02_mod12_post0\n#> 3 1666 30 0.00750 124 19 rsq standard 0.943 10 0.00309 pre20_mod21_post0\n#> 4 2000 24 0.00133 25 124 rsq standard 0.919 10 0.00383 pre09_mod25_post0\n#> 5 500 25 0.00316 52 8 rsq standard 0.899 10 0.00435 pre14_mod07_post0" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#chicago-l-train-data", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#chicago-l-train-data", + "title": "Case Study on Transportation", + "section": "Chicago L-Train data", + "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", - "title": "3 - Tuning Hyperparameters", - "section": "Choose a parameter combination ", - "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nlgbm_best <- select_best(lgbm_res, metric = \"mae\")\nlgbm_best\n#> # A tibble: 1 × 6\n#> trees min_n learn_rate `agent hash` `company hash` .config \n#> <int> <int> <dbl> <int> <int> <chr> \n#> 1 1250 9 0.1 107 80 pre19_mod16_post0" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#predictors", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#predictors", + "title": "Case Study on Transportation", + "section": "Predictors", + "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#checking-calibration", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#checking-calibration", - "title": "3 - Tuning Hyperparameters", - "section": "Checking Calibration ", - "text": "Checking Calibration \n\n\nlibrary(probably)\nlgbm_res |>\n collect_predictions(\n parameters = lgbm_best\n ) |>\n cal_plot_regression(\n truth = avg_price_per_room,\n estimate = .pred\n )" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#l-train-locations", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#l-train-locations", + "title": "Case Study on Transportation", + "section": "L Train Locations", + "text": "L Train Locations" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#running-in-parallel", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#running-in-parallel", - "title": "3 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\n\nlibrary(future)\nplan(multisession, workers = cores)\n\n# Now call `tune_grid()`!" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#your-turn-explore-the-data", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#your-turn-explore-the-data", + "title": "Case Study on Transportation", + "section": "Your turn: Explore the Data", + "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", - "title": "3 - Tuning Hyperparameters", - "section": "Running in parallel", - "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#splitting-with-chicago-data", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#splitting-with-chicago-data", + "title": "Case Study on Transportation", + "section": "Splitting with Chicago data ", + "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", - "title": "3 - Tuning Hyperparameters", - "section": "The ‘future’ of parallel processing", - "text": "The ‘future’ of parallel processing\nWe have relied on the foreach package for parallel processing.\nWe will start the transition to using the future package in the upcoming version of the tune package (version 1.3.0).\nThere will be a period of backward compatibility where you can still use foreach with future via the doFuture package. After that, the transition to future will occur.\nOverall, there will be minimal changes to your code." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#time-series-resampling", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#time-series-resampling", + "title": "Case Study on Transportation", + "section": "Time series resampling", + "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", - "title": "3 - Tuning Hyperparameters", - "section": "Early stopping for boosted trees", - "text": "Early stopping for boosted trees\nWe have directly optimized the number of trees as a tuning parameter.\nInstead we could\n\nSet the number of trees to a single large number.\nStop adding trees when performance gets worse.\n\nThis is known as “early stopping” and there is a parameter for that: stop_iter.\nEarly stopping has a potential to decrease the tuning time." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#rolling-forecast-origin-resampling", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#rolling-forecast-origin-resampling", + "title": "Case Study on Transportation", + "section": "Rolling forecast origin resampling", + "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn-2", - "href": "archive/2025-08-nyr/slides/advanced-03-tuning-hyperparameters.html#your-turn-2", - "title": "3 - Tuning Hyperparameters", - "section": "Your turn", - "text": "Your turn\n\n\nSet trees = 2000 and tune the stop_iter parameter.\nNote that you will need to regenerate lgbm_param with your new workflow!\n\n\n\n−+\n10:00" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#previously---setup", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#previously---setup", - "title": "5 - Iterative Search", - "section": "Previously - Setup", - "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates |> \n sample_n(5000) |> \n arrange(arrival_date) |> \n select(-arrival_date) |> \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-1", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-1", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#previously---data-usage", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#previously---data-usage", - "title": "5 - Iterative Search", - "section": "Previously - Data Usage", - "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-2", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-2", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#our-boosting-model", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#our-boosting-model", - "title": "5 - Iterative Search", - "section": "Our Boosting Model", - "text": "Our Boosting Model\nWe used feature hashing to generate a smaller set of indicator columns to deal with the large number of levels for the agent and country predictors.\n\nTree-based models (and a few others) don’t require indicators for categorical predictors. They can split on these variables as-is.\n\nWe’ll keep all categorical predictors as factors and focus on optimizing additional boosting parameters." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-3", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-3", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#our-boosting-model-1", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#our-boosting-model-1", - "title": "5 - Iterative Search", - "section": "Our Boosting Model", - "text": "Our Boosting Model\n\nlgbm_spec <- \n boost_tree(trees = 1000, learn_rate = tune(), min_n = tune(), \n tree_depth = tune(), loss_reduction = tune(), \n stop_iter = tune()) |> \n set_mode(\"regression\") |> \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(avg_price_per_room ~ ., lgbm_spec)\n\nlgbm_param <- \n lgbm_wflow |>\n extract_parameter_set_dials() |>\n update(learn_rate = learn_rate(c(-5, -1)))" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-4", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#times-series-resampling-4", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iterative-search", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iterative-search", - "title": "5 - Iterative Search", - "section": "Iterative Search", - "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our boosted tree.\n\nWe could do something like:\nmae_pred <- lm(mae ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#our-resampling-object", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#our-resampling-object", + "title": "Case Study on Transportation", + "section": "Our resampling object ", + "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iterative-search-1", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iterative-search-1", - "title": "5 - Iterative Search", - "section": "Iterative Search", - "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between MAE and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#feature-engineering-with-recipes", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#feature-engineering-with-recipes", + "title": "Case Study on Transportation", + "section": "Feature engineering with recipes ", + "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#a-large-grid", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#a-large-grid", - "title": "5 - Iterative Search", - "section": "A Large Grid", - "text": "A Large Grid" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe", + "title": "Case Study on Transportation", + "section": "A recipe", + "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#a-three-point-grid", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#a-three-point-grid", - "title": "5 - Iterative Search", - "section": "A Three Point Grid", - "text": "A Three Point Grid" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gaussian-processes-and-optimization", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gaussian-processes-and-optimization", - "title": "5 - Iterative Search", - "section": "Gaussian Processes and Optimization", - "text": "Gaussian Processes and Optimization\nWe can make a “meta-model” with a small set of historical performance results.\nGaussian Processes (GP) models are a good choice to model performance.\n\nIt is a Bayesian model so we are using Bayesian Optimization (BO).\nFor regression, we can assume that our data are multivariate normal.\nWe also define a covariance function for the variance relationship between data points. A common one is:\n\n\\[\\operatorname{cov}(\\boldsymbol{x}_i, \\boldsymbol{x}_j) = \\exp\\left(-\\frac{1}{2}|\\boldsymbol{x}_i - \\boldsymbol{x}_j|^2\\right) + \\sigma^2_{ij}\\]\n\nGPs are good because\n\nthey are flexible regression models (in the sense that splines are flexible).\nwe need to get mean and variance predictions (and they are Bayesian)\ntheir variability is based on spatial distances.\n\nSome people use random forests (with conformal variance estimates) or other methods but GPs are most popular." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-1", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-1", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#predicting-candidates", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#predicting-candidates", - "title": "5 - Iterative Search", - "section": "Predicting Candidates", - "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. MAE)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-2", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-2", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#your-turn", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#your-turn", - "title": "5 - Iterative Search", - "section": "Your turn", - "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize MAE.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---remove-constant-columns", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---remove-constant-columns", + "title": "Case Study on Transportation", + "section": "A recipe - remove constant columns ", + "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE) %>% \n step_zv(all_nominal_predictors())" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", - "title": "5 - Iterative Search", - "section": "GP Fit (ribbon is mean +/- 1SD)", - "text": "GP Fit (ribbon is mean +/- 1SD)" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---handle-correlations", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#a-recipe---handle-correlations", + "title": "Case Study on Transportation", + "section": "A recipe - handle correlations ", + "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#choosing-new-candidates", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#choosing-new-candidates", - "title": "5 - Iterative Search", - "section": "Choosing New Candidates", - "text": "Choosing New Candidates\nThis isn’t a very good fit but we can still use it.\nHow can we use the outputs to choose the next point to measure?\n\nAcquisition functions take the predicted mean and variance and use them to balance:\n\nexploration: new candidates should explore new areas.\nexploitation: new candidates must stay near existing values.\n\nExploration focuses on the variance, exploitation is about the mean." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#make-some-models", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#make-some-models", + "title": "Case Study on Transportation", + "section": "Make some models ", + "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) %>% set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) %>% \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#acquisition-functions", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#acquisition-functions", - "title": "5 - Iterative Search", - "section": "Acquisition Functions", - "text": "Acquisition Functions\nWe’ll use an acquisition function to select a new candidate.\nThe most popular method appears to be expected improvement (EI) above the current best results.\n\nZero at existing data points.\nThe expected improvement is integrated over all possible improvement (“expected” in the probability sense).\n\nWe would probably pick the point with the largest EI as the next point.\n(There are other functions beyond EI.)" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#process-them-on-the-resamples", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#process-them-on-the-resamples", + "title": "Case Study on Transportation", + "section": "Process them on the resamples", + "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set %>% \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement", - "title": "5 - Iterative Search", - "section": "Expected Improvement", - "text": "Expected Improvement" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#how-do-the-results-look", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#how-do-the-results-look", + "title": "Case Study on Transportation", + "section": "How do the results look?", + "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 31 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.798 0.104 16 recipe cubis… 1\n#> 2 pca_cubist Preprocessor3_Model3 mae 0.978 0.110 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor4_Model2 mae 0.983 0.122 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model1 mae 0.991 0.127 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor3_Model2 mae 0.991 0.113 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor2_Model2 mae 1.02 0.118 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor1_Model3 mae 1.05 0.134 16 recipe cubis… 7\n#> 8 basic_cubist Preprocessor1_Model8 mae 1.07 0.115 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model7 mae 1.07 0.112 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model6 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 21 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iteration", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#iteration", - "title": "5 - Iterative Search", - "section": "Iteration", - "text": "Iteration\nOnce we pick the candidate point, we measure performance for it (e.g. resampling).\n\nAnother GP is fit, EI is recomputed, and so on.\n\nWe stop when we have completed the allowed number of iterations or if we don’t see any improvement after a pre-set number of attempts." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#plot-the-results", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#plot-the-results", + "title": "Case Study on Transportation", + "section": "Plot the results ", + "text": "Plot the results \n\nautoplot(chi_res)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-fit-with-four-points", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-fit-with-four-points", - "title": "5 - Iterative Search", - "section": "GP Fit with four points", - "text": "GP Fit with four points" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#pull-out-specific-results", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#pull-out-specific-results", + "title": "Case Study on Transportation", + "section": "Pull out specific results ", + "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n autoplot()" }, - { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement-1", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement-1", - "title": "5 - Iterative Search", - "section": "Expected Improvement", - "text": "Expected Improvement" + { + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-evolution", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#gp-evolution", - "title": "5 - Iterative Search", - "section": "GP Evolution", - "text": "GP Evolution" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-1", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-1", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement-evolution", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#expected-improvement-evolution", - "title": "5 - Iterative Search", - "section": "Expected Improvement Evolution", - "text": "Expected Improvement Evolution" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-2", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-2", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#bo-in-tidymodels", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#bo-in-tidymodels", - "title": "5 - Iterative Search", - "section": "BO in tidymodels", - "text": "BO in tidymodels\nWe’ll use a function called tune_bayes() that has very similar syntax to tune_grid().\n\nIt has an additional initial argument for the initial set of performance estimates and parameter combinations for the GP model." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-3", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-3", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#initial-grid-points", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#initial-grid-points", - "title": "5 - Iterative Search", - "section": "Initial grid points", - "text": "Initial grid points\ninitial can be the results of another tune_*() function or an integer (in which case tune_grid() is used under to hood to make such an initial set of results).\n\nWe’ll run the optimization more than once, so let’s make an initial grid of results to serve as the substrate for the BO.\nI suggest at least the number of tuning parameters plus two as the initial grid for BO." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-4", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-4", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#an-initial-grid", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#an-initial-grid", - "title": "5 - Iterative Search", - "section": "An Initial Grid", - "text": "An Initial Grid\n\nreg_metrics <- metric_set(mae, rsq)\n\nset.seed(12)\ninit_res <-\n lgbm_wflow |>\n tune_grid(\n resamples = hotel_rs,\n grid = nrow(lgbm_param) + 2,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(init_res, metric = \"mae\") |> select(-.metric, -.estimator)\n#> # A tibble: 5 × 9\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config \n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> \n#> 1 33 12 0.0215 8.25e- 9 5 10.1 10 0.182 Preprocessor1_Model6\n#> 2 14 10 0.1 3.16e+ 1 8 10.2 10 0.192 Preprocessor1_Model3\n#> 3 8 3 0.00464 1 e-10 11 14.4 10 0.305 Preprocessor1_Model2\n#> 4 40 1 0.001 4.64e- 3 14 36.1 10 0.408 Preprocessor1_Model7\n#> 5 27 15 0.000215 6.81e- 7 20 44.5 10 0.371 Preprocessor1_Model5" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-5", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-5", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#bo-using-tidymodels", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#bo-using-tidymodels", - "title": "5 - Iterative Search", - "section": "BO using tidymodels", - "text": "BO using tidymodels\n\nctrl_bo <- control_bayes(verbose_iter = TRUE) # <- for demonstration\n\nset.seed(15)\nlgbm_bayes_res <-\n lgbm_wflow |>\n tune_bayes(\n resamples = hotel_rs,\n initial = init_res, # <- initial results\n iter = 20,\n param_info = lgbm_param,\n control = ctrl_bo,\n metrics = reg_metrics\n )\n#> Optimizing mae using the expected improvement\n#> \n#> ── Iteration 1 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=10.05 (@iter 0)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=7, tree_depth=13, learn_rate=0.0802, loss_reduction=1.13e-10, stop_iter=17\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.599 (+/-0.135)\n#> \n#> ── Iteration 2 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=7, learn_rate=0.0255, loss_reduction=1.22e-07, stop_iter=14\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.895 (+/-0.183)\n#> \n#> ── Iteration 3 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=28, tree_depth=5, learn_rate=0.0507, loss_reduction=1.91e-09, stop_iter=16\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.975 (+/-0.186)\n#> \n#> ── Iteration 4 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=3, tree_depth=11, learn_rate=0.0117, loss_reduction=0.00846, stop_iter=18\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.25 (+/-0.194)\n#> \n#> ── Iteration 5 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=34, tree_depth=1, learn_rate=0.0695, loss_reduction=0.00028, stop_iter=10\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=15.57 (+/-0.254)\n#> \n#> ── Iteration 6 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=18, tree_depth=13, learn_rate=0.0229, loss_reduction=0.181, stop_iter=19\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.882 (+/-0.166)\n#> \n#> ── Iteration 7 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=39, tree_depth=11, learn_rate=0.0485, loss_reduction=0.000906, stop_iter=19\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.883 (+/-0.169)\n#> \n#> ── Iteration 8 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=27, tree_depth=11, learn_rate=0.0396, loss_reduction=1.54e-10, stop_iter=6\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.811 (+/-0.195)\n#> \n#> ── Iteration 9 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=39, tree_depth=5, learn_rate=0.0131, loss_reduction=6.45e-05, stop_iter=16\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.55 (+/-0.152)\n#> \n#> ── Iteration 10 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=15, tree_depth=15, learn_rate=0.00677, loss_reduction=5.01e-06, stop_iter=19\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.73 (+/-0.214)\n#> \n#> ── Iteration 11 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.599 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=15, tree_depth=15, learn_rate=0.0929, loss_reduction=2.08, stop_iter=18\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.82 (+/-0.141)\n#> ! No improvement for 10 iterations; returning current results." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#building-a-model-stack", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#building-a-model-stack", + "title": "Case Study on Transportation", + "section": "Building a model stack ", + "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#best-results", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#best-results", - "title": "5 - Iterative Search", - "section": "Best results", - "text": "Best results\n\nshow_best(lgbm_bayes_res, metric = \"mae\") |> select(-.metric, -.estimator)\n#> # A tibble: 5 × 10\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config .iter\n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> <int>\n#> 1 7 13 0.0802 1.13e-10 17 9.60 10 0.135 Iter1 1\n#> 2 27 11 0.0396 1.54e-10 6 9.81 10 0.195 Iter8 8\n#> 3 15 15 0.0929 2.08e+ 0 18 9.82 10 0.141 Iter11 11\n#> 4 18 13 0.0229 1.81e- 1 19 9.88 10 0.166 Iter6 6\n#> 5 39 11 0.0485 9.06e- 4 19 9.88 10 0.169 Iter7 7" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#start-the-stack-and-add-members", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#start-the-stack-and-add-members", + "title": "Case Study on Transportation", + "section": "Start the stack and add members ", + "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() %>% \n add_candidates(chi_res)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results", - "title": "5 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\")" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#estimate-weights-for-each-candidate", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#estimate-weights-for-each-candidate", + "title": "Case Study on Transportation", + "section": "Estimate weights for each candidate ", + "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack)\n\nchi_stack_res\n#> # A tibble: 5 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.343\n#> 2 pca_cubist_3_2 cubist_rules 0.236\n#> 3 basic_cubist_1_4 cubist_rules 0.189\n#> 4 pca_lm_4_1 linear_reg 0.163\n#> 5 pca_cubist_3_3 cubist_rules 0.109" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results-1", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results-1", - "title": "5 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"parameters\")" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#how-did-it-do", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#how-did-it-do", + "title": "Case Study on Transportation", + "section": "How did it do? ", + "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results-2", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#plotting-bo-results-2", - "title": "5 - Iterative Search", - "section": "Plotting BO Results", - "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\")" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#what-does-it-use", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#what-does-it-use", + "title": "Case Study on Transportation", + "section": "What does it use? ", + "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#enhance", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#enhance", - "title": "5 - Iterative Search", - "section": "ENHANCE", - "text": "ENHANCE\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\") +\n ylim(c(9, 14))" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#fit-the-required-candidate-models", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#fit-the-required-candidate-models", + "title": "Case Study on Transportation", + "section": "Fit the required candidate models", + "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#your-turn-1", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#your-turn-1", - "title": "5 - Iterative Search", - "section": "Your turn", - "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\nBonus points: Before the optimization is done, press <esc> and see what happens.\n\n\n\n−+\n10:00" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#the-test-set-best-cubist-model", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#the-test-set-best-cubist-model", + "title": "Case Study on Transportation", + "section": "The test set: best Cubist model ", + "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n select_best()\n\ncubist_res <- \n chi_res %>% \n extract_workflow(\"pca_cubist\") %>% \n finalize_workflow(best_cubist) %>% \n last_fit(split = chi_split, metrics = metric_set(mae))" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#notes", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#notes", - "title": "5 - Iterative Search", - "section": "Notes", - "text": "Notes\n\nStopping tune_bayes() will return the current results.\nParallel processing can still be used to more efficiently measure each candidate point.\nThere are a lot of other iterative methods that you can use.\nThe finetune package also has functions for simulated annealing search." + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#the-test-set-stack-ensemble", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#the-test-set-stack-ensemble", + "title": "Case Study on Transportation", + "section": "The test set: stack ensemble", + "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) %>% \n bind_cols(chi_test)" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#finalizing-the-model", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#finalizing-the-model", - "title": "5 - Iterative Search", - "section": "Finalizing the Model", - "text": "Finalizing the Model\nLet’s say that we’ve tried a lot of different models and we like our lightgbm model the most.\nWhat do we do now?\n\nFinalize the workflow by choosing the values for the tuning parameters.\nFit the model on the entire training set.\nVerify performance using the test set.\nDocument and publish the model(?)" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#compare-the-results", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#compare-the-results", + "title": "Case Study on Transportation", + "section": "Compare the results ", + "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.670 Preprocessor1_Model1\n\nstack_pred %>% mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.689" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#locking-down-the-tuning-parameters", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#locking-down-the-tuning-parameters", - "title": "5 - Iterative Search", - "section": "Locking Down the Tuning Parameters", - "text": "Locking Down the Tuning Parameters\nWe can take the results of the Bayesian optimization and accept the best results:\n\nbest_param <- select_best(lgbm_bayes_res, metric = \"mae\")\nfinal_wflow <- \n lgbm_wflow |> \n finalize_workflow(best_param)\nfinal_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: boost_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> avg_price_per_room ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Boosted Tree Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> min_n = 7\n#> tree_depth = 13\n#> learn_rate = 0.0801616272924067\n#> loss_reduction = 1.13369097175613e-10\n#> stop_iter = 17\n#> \n#> Engine-Specific Arguments:\n#> num_threads = 1\n#> \n#> Computational engine: lightgbm" + "objectID": "archive/2023-09-posit-conf/extras-transit-case-study.html#plot-the-test-set", + "href": "archive/2023-09-posit-conf/extras-transit-case-study.html#plot-the-test-set", + "title": "Case Study on Transportation", + "section": "Plot the test set ", + "text": "Plot the test set \n\n\nlibrary(probably)\ncubist_res %>% \n collect_predictions() %>% \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#the-final-fit", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#the-final-fit", - "title": "5 - Iterative Search", - "section": "The Final Fit", - "text": "The Final Fit\nWe can use individual functions:\nfinal_fit <- final_wflow |> fit(data = hotel_train)\n\n# then predict() or augment() \n# then compute metrics\n\nRemember that there is also a convenience function to do all of this:\n\nset.seed(3893)\nfinal_res <- final_wflow |> last_fit(hotel_split, metrics = reg_metrics)\nfinal_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [3749/1251]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [1,251 × 4]> <workflow>" + "objectID": "archive/2023-09-posit-conf/index.html", + "href": "archive/2023-09-posit-conf/index.html", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels offered at posit::conf(2023). The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to tidymodels and Advanced tidymodels courses.\nIntroduction to tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and basic pre-processing with recipes. Time permitting, you’ll be introduced to model optimization using the tune package. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nAdvanced tidymodels will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." }, { - "objectID": "archive/2025-08-nyr/slides/advanced-05-iterative.html#test-set-results", - "href": "archive/2025-08-nyr/slides/advanced-05-iterative.html#test-set-results", - "title": "5 - Iterative Search", - "section": "Test Set Results", - "text": "Test Set Results\n\n\n\nfinal_res |> \n collect_predictions() |> \n cal_plot_regression(\n truth = avg_price_per_room, \n estimate = .pred)\n\nTest set performance:\n\nfinal_res |> collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 9.68 Preprocessor1_Model1\n#> 2 rsq standard 0.946 Preprocessor1_Model1\n\n\n\n\n\n\n\n\n\n\n\n\nRecall that resampling predicted the MAE to be 9.599." + "objectID": "archive/2023-09-posit-conf/index.html#welcome", + "href": "archive/2023-09-posit-conf/index.html#welcome", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels offered at posit::conf(2023). The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to tidymodels and Advanced tidymodels courses.\nIntroduction to tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and basic pre-processing with recipes. Time permitting, you’ll be introduced to model optimization using the tune package. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nAdvanced tidymodels will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#section", - "href": "archive/2025-08-nyr/slides/annotations.html#section", - "title": "Annotations", - "section": "👀", - "text": "👀\nThis page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." + "objectID": "archive/2023-09-posit-conf/index.html#is-this-workshop-for-me", + "href": "archive/2023-09-posit-conf/index.html#is-this-workshop-for-me", + "title": "Machine learning with tidymodels", + "section": "Is this workshop for me? ", + "text": "Is this workshop for me? \nDepending on your background, one of Introduction to tidymodels or Advanced tidymodels might serve you better than the other.\n\nIntroduction to tidymodels\nThis workshop is for you if you:\n\nare comfortable using tidyverse packages to read data into R, transform and reshape data, and make a variety of graphs, and\nhave had some exposure to basic statistical concepts such as linear models, residuals, etc.\n\nIntermediate or expert familiarity with modeling or machine learning is not required. Interested students who have intermediate or expert familiarity with modeling or machine learning may be interested in the Advanced tidymodels workshop.\n\n\nAdvanced tidymodels\nThis workshop is for you if you:\n\nhave the prerequisite skills listed for the Introduction to tidymodels workshops,\nhave used tidymodels packages like recipes, rsample, and parsnip, and\nhave some experience with evaluating statistical models using resampling techniques like v-fold cross-validation or the bootstrap.\n\nParticipants who are new to tidymodels or machine learning will benefit from taking the Introduction to tidymodels workshop before joining this one. Participants who have completed the “Introduction to tidymodels” workshop will be well-prepared for this course." }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#data-splitting-and-spending", - "href": "archive/2025-08-nyr/slides/annotations.html#data-splitting-and-spending", - "title": "Annotations", - "section": "Data splitting and spending", - "text": "Data splitting and spending\nMore about the initial data split can be found in Chapter 3 of Applied Machine Learning for Tabular Data (AML4TD).\nIn particular, a three-way split into training, validation, and testing set can be done via\n\nset.seed(123)\ninitial_validation_split(forested, prop = c(0.6, 0.2))\n#> <Training/Validation/Testing/Total>\n#> <4264/1421/1422/7107>" + "objectID": "archive/2023-09-posit-conf/index.html#preparation", + "href": "archive/2023-09-posit-conf/index.html#preparation", + "title": "Machine learning with tidymodels", + "section": "Preparation", + "text": "Preparation\nThe process to set up your computer for either workshop will look the same. Please join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2022.02), available at https://posit.co/download/rstudio-desktop/\nThe following R packages, which you can install from the R console:\n\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\",\n \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\",\n \"splines2\", \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \n \"vetiver\", \"remotes\")\n\ninstall.packages(pkgs)\n\nIf you’re a Windows user and encounter an error message during installation noting a missing Rtools installation, install Rtools using the installer linked here." }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#what-is-set.seed", - "href": "archive/2025-08-nyr/slides/annotations.html#what-is-set.seed", - "title": "Annotations", - "section": "What is set.seed()?", - "text": "What is set.seed()?\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" + "objectID": "archive/2023-09-posit-conf/index.html#slides", + "href": "archive/2023-09-posit-conf/index.html#slides", + "title": "Machine learning with tidymodels", + "section": "Slides", + "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. There are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to tidymodels\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n05: Tuning models\n06: Wrapping up\n\n\n\nAdvanced tidymodels\n\n01: Introduction\n02: Feature engineering using recipes\n03: Tuning hyperparameters (grid search)\n04: Grid search via racing\n05: Iterative search\n06: Wrapping up\n\n\n\nExtra content (time permitting)\n\nIntro: Time-based splitting\nIntro: Using workflowsets\nIntro: Using recipes\nAdvanced: Transit case study (includes stacking)\nAdvanced: Effect encoding\nAdvanced: Model deployment\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#what-is-wrong-with-this", - "href": "archive/2025-08-nyr/slides/annotations.html#what-is-wrong-with-this", - "title": "Annotations", - "section": "What is wrong with this?", - "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" + "objectID": "archive/2023-09-posit-conf/index.html#code", + "href": "archive/2023-09-posit-conf/index.html#code", + "title": "Machine learning with tidymodels", + "section": "Code", + "text": "Code\nQuarto files for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#brier-score", - "href": "archive/2025-08-nyr/slides/annotations.html#brier-score", - "title": "Annotations", - "section": "Brier score", - "text": "Brier score\nThe Brier score measures how close a model probability estimate is to its best possible value (i.e., zero or one).\nIn the best case, the model is perfect, and every prediction equals 0.0 or 1.0 (depending on the true class). In this case, the Brier score is zero.\nWhen the model is uninformative and there are two classes, the worst-case values range from 0.25 to about 0.50. Imagine that the model predicts the same noninformative prediction of 50% (basically “¯\\(ツ)/¯”). In that case, every prediction is either \\((0.00 - 0.50)^2\\) or \\((1.00 - 0.50)^2\\). The average of those is 0.25.\nThere are many different ways a model can be bad though, and some of these will produce Brier scores between 0.25 and 0.50." + "objectID": "archive/2023-09-posit-conf/index.html#past-workshops", + "href": "archive/2023-09-posit-conf/index.html#past-workshops", + "title": "Machine learning with tidymodels", + "section": "Past workshops", + "text": "Past workshops\n\nJuly 2022 at rstudio::conf()\nAugust 2022 in Reykjavik\nJuly 2023 at the New York R Conference" }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#dangers-of-overfitting", - "href": "archive/2025-08-nyr/slides/annotations.html#dangers-of-overfitting", - "title": "Annotations", - "section": "Dangers of overfitting", - "text": "Dangers of overfitting\nSee the “Overfitting” chapter of AML4TD for more information." + "objectID": "archive/2023-09-posit-conf/index.html#acknowledgments", + "href": "archive/2023-09-posit-conf/index.html#acknowledgments", + "title": "Machine learning with tidymodels", + "section": "Acknowledgments", + "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#where-are-the-fitted-models", - "href": "archive/2025-08-nyr/slides/annotations.html#where-are-the-fitted-models", - "title": "Annotations", - "section": "Where are the fitted models?", - "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" + "objectID": "archive/2023-09-posit-conf/index.html#reuse-and-licensing", + "href": "archive/2023-09-posit-conf/index.html#reuse-and-licensing", + "title": "Machine learning with tidymodels", + "section": "Reuse and licensing", + "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#resampling-strategy", - "href": "archive/2025-08-nyr/slides/annotations.html#resampling-strategy", - "title": "Annotations", - "section": "Resampling Strategy", - "text": "Resampling Strategy\nThese data have a time component, and while not a typical time series data set, we have the option to use a time series resampling method.\nAn example is shown in the extra slides “Case Study on Transportation”.\nConsider the agent data. Cross-validation may not group all of an agent’s data into the analysis or assessment sets. In this case, our analysis data might have future data that is later than the agent’s data in the assessment set." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips", + "title": "2 - Your data budget", + "section": "Data on Chicago taxi trips", + "text": "Data on Chicago taxi trips\n\n\n\nThe city of Chicago releases anonymized trip-level data on taxi trips in the city.\nWe pulled a sample of 10,000 rides occurring in early 2022.\nType ?taxi to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/8322/taxi" }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#different-types-of-grids", - "href": "archive/2025-08-nyr/slides/annotations.html#different-types-of-grids", - "title": "Annotations", - "section": "Different types of grids", - "text": "Different types of grids\nMore on space-filling designs in Chapters 4 and 5 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences.\nIn the next version of tune (version 1.3.0) an improved set of space-filling designs will be the first choice if you ask for an automated grid." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips-1", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips-1", + "title": "2 - Your data budget", + "section": "Data on Chicago taxi trips", + "text": "Data on Chicago taxi trips\n\n\n\nN = 10,000\nA nominal outcome, tip, with levels \"yes\" and \"no\"\nSeveral nominal variables like pickup & dropoff location, taxi ID, and payment type.\nSeveral numeric variables like trip length and fare subtotals.\n\n\n\n\n\nCredit: https://unsplash.com/photos/7_r85l4eht8\n\n\n“Fare subtotals” refers to the fare itself, tax, tolls, tip amount.\nActual variables in our data:\ntip: Whether the rider left a tip. A factor with levels “yes” and “no”.\ndistance: The trip distance, in odometer miles.\ncompany: The taxi company, as a factor. Companies that occurred few times were binned as “other”.\nlocal: Whether the trip started in the same community area as it began. See the source data for community area values.\ndow: The day of the week in which the trip began, as a factor.\nmonth: The month in which the trip began, as a factor.\nhour: The hour of the day in which the trip began, as a numeric." }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#update-parameter-ranges", - "href": "archive/2025-08-nyr/slides/annotations.html#update-parameter-ranges", - "title": "Annotations", - "section": "Update parameter ranges", - "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object |> \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) |> \n step_spline_natural(dis, deg_free = tune()) |> \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#checklist-for-predictors", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#checklist-for-predictors", + "title": "2 - Your data budget", + "section": "Checklist for predictors", + "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?" }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#early-stopping-for-boosted-trees", - "href": "archive/2025-08-nyr/slides/annotations.html#early-stopping-for-boosted-trees", - "title": "Annotations", - "section": "Early stopping for boosted trees", - "text": "Early stopping for boosted trees\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the lightgbm package.\nbonsai has its own wrapper around (lightgbm::lgb.train()) called bonsai::train_lightgbm(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the lightgbm entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for lightgbm)." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips-2", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-on-chicago-taxi-trips-2", + "title": "2 - Your data budget", + "section": "Data on Chicago taxi trips", + "text": "Data on Chicago taxi trips\n\nlibrary(tidymodels)\n\ntaxi\n#> # A tibble: 10,000 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 17.2 Chicago Independents no Thu Feb 16\n#> 2 yes 0.88 City Service yes Thu Mar 8\n#> 3 yes 18.1 other no Mon Feb 18\n#> 4 yes 20.7 Chicago Independents no Mon Apr 8\n#> 5 yes 12.2 Chicago Independents no Sun Mar 21\n#> 6 yes 0.94 Sun Taxi yes Sat Apr 23\n#> 7 yes 17.5 Flash Cab no Fri Mar 12\n#> 8 yes 17.7 other no Sun Jan 6\n#> 9 yes 1.85 Taxicab Insurance Agency Llc no Fri Apr 12\n#> 10 yes 1.47 City Service no Tue Mar 14\n#> # ℹ 9,990 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#gaussian-processes-and-optimization", - "href": "archive/2025-08-nyr/slides/annotations.html#gaussian-processes-and-optimization", - "title": "Annotations", - "section": "Gaussian Processes and Optimization", - "text": "Gaussian Processes and Optimization\nSome other references for GP’s:\n\nChapter 5 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences\nBayesian Optimization, Chapter 3 (pdf)\nGaussian Processes for Machine Learning (pdf)" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#acquisition-functions", - "href": "archive/2025-08-nyr/slides/annotations.html#acquisition-functions", - "title": "Annotations", - "section": "Acquisition Functions", - "text": "Acquisition Functions\nMore references:\n\nChapter 7 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences\nBayesian Optimization, Chapter 6 (pdf)\nGaussian Processes for Machine Learning" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-1", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-1", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending" }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#per-agent-statistics", - "href": "archive/2025-08-nyr/slides/annotations.html#per-agent-statistics", - "title": "Annotations", - "section": "Per-agent statistics", - "text": "Per-agent statistics\nThe effect encoding method essentially takes the effect of a variable, like agent, and makes a data column for that effect. In our example, affect of the agent on the ADR is quantified by a model and then added as a data column to be used in the model.\nSuppose agent Max has a single reservation in the data and it had an ADR of €200. If we used a naive estimate for Max’s effect, the model is being told that Max should always produce an effect of €200. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned agent Davis, who has taken 250 reservations with an average ADR of €100. Davis’s mean is more predictive because it is estimated with better data (i.e., more total reservations). Partial pooling leverages the entire data set and can borrow strength from all of the agents. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a agent’s data is of good quality, the partial pooling effect estimate is closer to the raw mean. Max’s data is not great and is “shrunk” towards the center of the overall average. Since there is so little known about Max’s reservation history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nSince this example has a numeric outcome, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-2", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-2", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#agent-effects", - "href": "archive/2025-08-nyr/slides/annotations.html#agent-effects", - "title": "Annotations", - "section": "Agent effects", - "text": "Agent effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage though." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2025-08-nyr/slides/annotations.html#a-recipe---handle-correlations", - "href": "archive/2025-08-nyr/slides/annotations.html#a-recipe---handle-correlations", - "title": "Annotations", - "section": "A recipe - handle correlations", - "text": "A recipe - handle correlations\nIn this code chunk, What’s the story with !!stations?\n\nchi_pca_rec <- \n chi_rec |> \n step_normalize(all_of(!!stations)) |> \n step_pca(all_of(!!stations), num_comp = tune())\n\nstations is a vector of names of 20 columns that we want to use in the steps. If the list were shorter, we could type them in (e.g., c(\"col1\", \"col2\") etc.).\nThe vector lives in our global workspace, and if we are in parallel, the worker processes might not have access to stations. The !! (frequently said as “bang bang”) inserts the actual contents of the vector into the all_of() calls so that it looks like you just typed it in.\nThis means that the parallel process workers have a copy of the data in their reach and the code will run without error." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-initial-split", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-initial-split", + "title": "2 - Your data budget", + "section": "The initial split ", + "text": "The initial split \n\nset.seed(123)\ntaxi_split <- initial_split(taxi)\ntaxi_split\n#> <Training/Testing/Total>\n#> <7500/2500/10000>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data We will talk about more powerful ways of splitting, like stratification, later" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#chicago-l-train-data", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#chicago-l-train-data", - "title": "Case Study on Transportation", - "section": "Chicago L-Train data", - "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#what-is-set.seed", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#what-is-set.seed", + "title": "2 - Your data budget", + "section": "What is set.seed()?", + "text": "What is set.seed()?\nTo create that split of the data, R generates “pseudo-random” numbers: while they are made to behave like random numbers, their generation is deterministic give a “seed”.\nThis allows us to reproduce results by setting that seed.\nWhich seed you pick doesn’t matter, as long as you don’t try a bunch of seeds and pick the one that gives you the best performance." }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#predictors", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#predictors", - "title": "Case Study on Transportation", - "section": "Predictors", - "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#accessing-the-data", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#accessing-the-data", + "title": "2 - Your data budget", + "section": "Accessing the data ", + "text": "Accessing the data \n\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#l-train-locations", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#l-train-locations", - "title": "Case Study on Transportation", - "section": "L Train Locations", - "text": "L Train Locations" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-training-set", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-training-set", + "title": "2 - Your data budget", + "section": "The training set", + "text": "The training set\n\ntaxi_train\n#> # A tibble: 7,500 × 7\n#> tip distance company local dow month hour\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes 0.7 Taxi Affiliation Services yes Tue Mar 18\n#> 2 yes 0.99 Sun Taxi yes Tue Jan 8\n#> 3 yes 1.78 other no Sat Mar 22\n#> 4 yes 0 Taxi Affiliation Services yes Wed Apr 15\n#> 5 yes 0 Taxi Affiliation Services no Sun Jan 21\n#> 6 yes 2.3 other no Sat Apr 21\n#> 7 yes 6.35 Sun Taxi no Wed Mar 16\n#> 8 yes 2.79 other no Sun Feb 14\n#> 9 yes 16.6 other no Sun Apr 18\n#> 10 yes 0.02 Chicago Independents yes Sun Apr 15\n#> # ℹ 7,490 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#your-turn-explore-the-data", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#your-turn-explore-the-data", - "title": "Case Study on Transportation", - "section": "Your turn: Explore the Data", - "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n−+\n05:00" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-test-set", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-test-set", + "title": "2 - Your data budget", + "section": "The test set ", + "text": "The test set \n🙈\n\nThere are 2500 rows and 7 columns in the test set." }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#splitting-with-chicago-data", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#splitting-with-chicago-data", - "title": "Case Study on Transportation", - "section": "Splitting with Chicago data ", - "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn-1", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn-1", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#time-series-resampling", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#time-series-resampling", - "title": "Case Study on Transportation", - "section": "Time series resampling", - "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-3", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-3", + "title": "2 - Your data budget", + "section": "Data splitting and spending ", + "text": "Data splitting and spending \n\nset.seed(123)\ntaxi_split <- initial_split(taxi, prop = 0.8)\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)\n\nnrow(taxi_train)\n#> [1] 8000\nnrow(taxi_test)\n#> [1] 2000" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#rolling-forecast-origin-resampling", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#rolling-forecast-origin-resampling", - "title": "Case Study on Transportation", - "section": "Rolling forecast origin resampling", - "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#validation-set", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#validation-set", + "title": "2 - Your data budget", + "section": "Validation set", + "text": "Validation set\n\nset.seed(123)\ninitial_validation_split(taxi, prop = c(0.6, 0.2))\n#> <Training/Validation/Testing/Total>\n#> <6000/2000/2000/10000>" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn-2", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#your-turn-2", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nExplore the taxi_train data on your own!\n\nWhat’s the distribution of the outcome, tip?\nWhat’s the distribution of numeric variables like distance?\nHow does tip differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-1", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-1", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-2", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-2", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n ggplot(aes(x = tip)) +\n geom_bar()" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-2", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-2", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-3", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-3", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n ggplot(aes(x = tip, fill = local)) +\n geom_bar() +\n scale_fill_viridis_d(end = .5)" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-3", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-3", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-4", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-4", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n ggplot(aes(x = hour, fill = tip)) +\n geom_bar()" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-4", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#times-series-resampling-4", - "title": "Case Study on Transportation", - "section": "Times series resampling ", - "text": "Times series resampling \n\nchi_rs <-\n chi_train |>\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] |> assessment() |> pluck(\"date\") |> range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] |> assessment() |> pluck(\"date\") |> range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-5", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-5", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n ggplot(aes(x = hour, fill = tip)) +\n geom_bar(position = \"fill\")" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#our-resampling-object", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#our-resampling-object", - "title": "Case Study on Transportation", - "section": "Our resampling object ", - "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-6", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-6", + "title": "2 - Your data budget", + "section": "", + "text": "taxi_train %>% \n ggplot(aes(x = distance)) +\n geom_histogram(bins = 100) +\n facet_grid(vars(tip))" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#feature-engineering-with-recipes", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#feature-engineering-with-recipes", - "title": "Case Study on Transportation", - "section": "Feature engineering with recipes ", - "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-7", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#section-7", + "title": "2 - Your data budget", + "section": "", + "text": "Stratified sampling would split within response values\n\nBased on our EDA, we know that the source data contains fewer \"no\" tip values than \"yes\". We want to make sure we allot equal proportions of those responses so that both the training and testing data have enough of each to give accurate estimates." }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe", - "title": "Case Study on Transportation", - "section": "A recipe", - "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#stratification", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#stratification", + "title": "2 - Your data budget", + "section": "Stratification", + "text": "Stratification\nUse strata = tip\n\nset.seed(123)\ntaxi_split <- initial_split(taxi, prop = 0.8, strata = tip)\ntaxi_split\n#> <Training/Testing/Total>\n#> <8000/2000/10000>" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) |> \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#stratification-1", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#stratification-1", + "title": "2 - Your data budget", + "section": "Stratification", + "text": "Stratification\nStratification often helps, with very little downside" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates-1", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates-1", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) |> \n step_date(date, features = c(\"dow\", \"month\", \"year\")) |> \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." + "objectID": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-whole-game---status-update", + "href": "archive/2023-09-posit-conf/intro-02-data-budget.html#the-whole-game---status-update", + "title": "2 - Your data budget", + "section": "The whole game - status update", + "text": "The whole game - status update" + }, + { + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#looking-at-predictions", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#looking-at-predictions", + "title": "4 - Evaluating models", + "section": "Looking at predictions", + "text": "Looking at predictions\n\naugment(taxi_fit, new_data = taxi_train) %>%\n relocate(tip, .pred_class, .pred_yes, .pred_no)\n#> # A tibble: 8,000 × 10\n#> tip .pred_class .pred_yes .pred_no distance company local dow month hour\n#> <fct> <fct> <dbl> <dbl> <dbl> <fct> <fct> <fct> <fct> <int>\n#> 1 yes yes 0.967 0.0333 17.2 Chicag… no Thu Feb 16\n#> 2 yes yes 0.935 0.0646 0.88 City S… yes Thu Mar 8\n#> 3 yes yes 0.967 0.0333 18.1 other no Mon Feb 18\n#> 4 yes yes 0.949 0.0507 12.2 Chicag… no Sun Mar 21\n#> 5 yes yes 0.821 0.179 0.94 Sun Ta… yes Sat Apr 23\n#> 6 yes yes 0.967 0.0333 17.5 Flash … no Fri Mar 12\n#> 7 yes yes 0.967 0.0333 17.7 other no Sun Jan 6\n#> 8 yes yes 0.938 0.0616 1.85 Taxica… no Fri Apr 12\n#> 9 yes yes 0.938 0.0616 0.53 Sun Ta… no Tue Mar 18\n#> 10 yes yes 0.931 0.0694 6.65 Taxica… no Sun Apr 11\n#> # ℹ 7,990 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates-2", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---work-with-dates-2", - "title": "Case Study on Transportation", - "section": "A recipe - work with dates ", - "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) |> \n step_date(date, features = c(\"dow\", \"month\", \"year\")) |> \n step_holiday(date) |> \n update_role(date, new_role = \"id\") |>\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points. The help page has a good discussion about the nuances." + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---remove-constant-columns", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---remove-constant-columns", - "title": "Case Study on Transportation", - "section": "A recipe - remove constant columns ", - "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) |> \n step_date(date, features = c(\"dow\", \"month\", \"year\")) |> \n step_holiday(date) |> \n update_role(date, new_role = \"id\") |>\n update_role_requirements(role = \"id\", bake = TRUE) |> \n step_zv(all_nominal_predictors())" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix-1", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(taxi_fit, new_data = taxi_train) %>%\n conf_mat(truth = tip, estimate = .pred_class)\n#> Truth\n#> Prediction yes no\n#> yes 7341 536\n#> no 43 80" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---handle-correlations", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#a-recipe---handle-correlations", - "title": "Case Study on Transportation", - "section": "A recipe - handle correlations ", - "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec |> \n step_normalize(all_of(!!stations)) |> \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix-2", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#confusion-matrix-2", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(taxi_fit, new_data = taxi_train) %>%\n conf_mat(truth = tip, estimate = .pred_class) %>%\n autoplot(type = \"heatmap\")" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#make-some-models", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#make-some-models", - "title": "Case Study on Transportation", - "section": "Make some models ", - "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) |> set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) |> \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n accuracy(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#process-them-on-the-resamples", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#process-them-on-the-resamples", - "title": "Case Study on Transportation", - "section": "Process them on the resamples", - "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set |> \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-accuracy", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-accuracy", + "title": "4 - Evaluating models", + "section": "Dangers of accuracy ", + "text": "Dangers of accuracy \nWe need to be careful of using accuracy() since it can give “good” performance by only predicting one way with imbalanced data\n\naugment(taxi_fit, new_data = taxi_train) %>%\n mutate(.pred_class = factor(\"yes\", levels = c(\"yes\", \"no\"))) %>%\n accuracy(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.923" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#how-do-the-results-look", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#how-do-the-results-look", - "title": "Case Study on Transportation", - "section": "How do the results look?", - "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 34 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.804 0.101 16 recipe cubis… 1\n#> 2 basic_cubist Preprocessor1_Model1 mae 0.901 0.115 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor2_Model2 mae 0.985 0.120 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model2 mae 0.990 0.121 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor1_Model3 mae 1.00 0.119 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor4_Model1 mae 1.02 0.131 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor3_Model3 mae 1.03 0.121 16 recipe cubis… 7\n#> 8 pca_cubist Preprocessor3_Model2 mae 1.07 0.137 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model9 mae 1.07 0.115 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model8 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 24 more rows" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-1", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n sensitivity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.994" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#plot-the-results", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#plot-the-results", - "title": "Case Study on Transportation", - "section": "Plot the results ", - "text": "Plot the results \n\nautoplot(chi_res)" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-2", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-2", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n sensitivity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.994\n\n\n\naugment(taxi_fit, new_data = taxi_train) %>%\n specificity(truth = tip, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.130" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#pull-out-specific-results", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#pull-out-specific-results", - "title": "Case Study on Transportation", - "section": "Pull out specific results ", - "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res |> \n extract_workflow_set_result(\"pca_cubist\") |> \n autoplot()" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-3", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-3", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_fit, new_data = taxi_train) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n#> 2 specificity binary 0.130\n#> 3 sensitivity binary 0.994" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-4", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-4", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\ntaxi_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(taxi_fit, new_data = taxi_train) %>%\n group_by(local) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 6 × 4\n#> local .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 yes accuracy binary 0.898\n#> 2 no accuracy binary 0.935\n#> 3 yes specificity binary 0.169\n#> 4 no specificity binary 0.116\n#> 5 yes sensitivity binary 0.987\n#> 6 no sensitivity binary 0.996" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-1", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-1", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#two-class-data", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#two-class-data", + "title": "4 - Evaluating models", + "section": "Two class data", + "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-2", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-2", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#varying-the-threshold", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#varying-the-threshold", + "title": "4 - Evaluating models", + "section": "Varying the threshold", + "text": "Varying the threshold" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-3", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-3", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curves", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curves", + "title": "4 - Evaluating models", + "section": "ROC curves", + "text": "ROC curves\nTo make an ROC (receiver operator characteristic) curve, we:\n\ncalculate the sensitivity and specificity for all possible thresholds\nplot false positive rate (x-axis) versus true positive rate (y-axis)\n\ngiven that sensitivity is the true positive rate, and specificity is the true negative rate. Hence 1 - specificity is the false positive rate.\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢\n\n\nROC curves are insensitive to class imbalance." }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-4", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-4", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curves-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curves-1", + "title": "4 - Evaluating models", + "section": "ROC curves ", + "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_curve(truth = tip, .pred_yes) %>%\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.783 0.209 0.981 \n#> 3 1 1 0.00135\n\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_auc(truth = tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.691" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-5", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#why-choose-just-one-final_fit-5", - "title": "Case Study on Transportation", - "section": "Why choose just one final_fit? ", - "text": "Why choose just one final_fit?" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curve-plot", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#roc-curve-plot", + "title": "4 - Evaluating models", + "section": "ROC curve plot ", + "text": "ROC curve plot \n\n\naugment(taxi_fit, new_data = taxi_train) %>% \n roc_curve(truth = tip, .pred_yes) %>%\n autoplot()" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#building-a-model-stack", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#building-a-model-stack", - "title": "Case Study on Transportation", - "section": "Building a model stack ", - "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#start-the-stack-and-add-members", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#start-the-stack-and-add-members", - "title": "Case Study on Transportation", - "section": "Start the stack and add members ", - "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() |> \n add_candidates(chi_res)" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-1", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#estimate-weights-for-each-candidate", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#estimate-weights-for-each-candidate", - "title": "Case Study on Transportation", - "section": "Estimate weights for each candidate ", - "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack, penalty = 10^seq(-6, -1, length.out = 25))\n\nchi_stack_res\n#> # A tibble: 5 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.354 \n#> 2 basic_cubist_1_1 cubist_rules 0.263 \n#> 3 pca_cubist_1_3 cubist_rules 0.203 \n#> 4 basic_cubist_1_6 cubist_rules 0.155 \n#> 5 pca_lm_2_1 linear_reg 0.0479" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-2", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-2", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#how-did-it-do", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#how-did-it-do", - "title": "Case Study on Transportation", - "section": "How did it do? ", - "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-3", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-3", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\ntaxi_fit %>%\n augment(taxi_train)\n#> # A tibble: 8,000 × 10\n#> tip distance company local dow month hour .pred_class .pred_yes .pred_no\n#> <fct> <dbl> <fct> <fct> <fct> <fct> <int> <fct> <dbl> <dbl>\n#> 1 yes 17.2 Chicag… no Thu Feb 16 yes 0.967 0.0333\n#> 2 yes 0.88 City S… yes Thu Mar 8 yes 0.935 0.0646\n#> 3 yes 18.1 other no Mon Feb 18 yes 0.967 0.0333\n#> 4 yes 12.2 Chicag… no Sun Mar 21 yes 0.949 0.0507\n#> 5 yes 0.94 Sun Ta… yes Sat Apr 23 yes 0.821 0.179 \n#> 6 yes 17.5 Flash … no Fri Mar 12 yes 0.967 0.0333\n#> 7 yes 17.7 other no Sun Jan 6 yes 0.967 0.0333\n#> 8 yes 1.85 Taxica… no Fri Apr 12 yes 0.938 0.0616\n#> 9 yes 0.53 Sun Ta… no Tue Mar 18 yes 0.938 0.0616\n#> 10 yes 6.65 Taxica… no Sun Apr 11 yes 0.931 0.0694\n#> # ℹ 7,990 more rows\n\nWe call this “resubstitution” or “repredicting the training set”" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#what-does-it-use", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#what-does-it-use", - "title": "Case Study on Transportation", - "section": "What does it use? ", - "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-4", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-4", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n\nWe call this a “resubstitution estimate”" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#fit-the-required-candidate-models", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#fit-the-required-candidate-models", - "title": "Case Study on Transportation", - "section": "Fit the required candidate models", - "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-5", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-5", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#the-test-set-best-cubist-model", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#the-test-set-best-cubist-model", - "title": "Case Study on Transportation", - "section": "The test set: best Cubist model ", - "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res |> \n extract_workflow_set_result(\"pca_cubist\") |> \n select_best()\n\ncubist_res <- \n chi_res |> \n extract_workflow(\"pca_cubist\") |> \n finalize_workflow(best_cubist) |> \n last_fit(split = chi_split, metrics = metric_set(mae))" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-6", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-6", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.928\n\n\n\ntaxi_fit %>%\n augment(taxi_test) %>%\n accuracy(tip, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.908\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#the-test-set-stack-ensemble", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#the-test-set-stack-ensemble", - "title": "Case Study on Transportation", - "section": "The test set: stack ensemble", - "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) |> \n bind_cols(chi_test)" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-1", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse augment() and a metric function to compute a classification metric like brier_class().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#compare-the-results", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#compare-the-results", - "title": "Case Study on Transportation", - "section": "Compare the results ", - "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.669 Preprocessor1_Model1\n\nstack_pred |> mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.669" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-7", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-7", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\ntaxi_fit %>%\n augment(taxi_train) %>%\n brier_class(tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0632\n\n\n\ntaxi_fit %>%\n augment(taxi_test) %>%\n brier_class(tip, .pred_yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0782\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" }, { - "objectID": "archive/2025-08-nyr/slides/extras-transit-case-study.html#plot-the-test-set", - "href": "archive/2025-08-nyr/slides/extras-transit-case-study.html#plot-the-test-set", - "title": "Case Study on Transportation", - "section": "Plot the test set ", - "text": "Plot the test set \n\n\nlibrary(probably)\ncubist_res |> \n collect_predictions() |> \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#who-are-you", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#who-are-you", - "title": "1 - Introduction", - "section": "Who are you?", - "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have some exposure to basic statistical concepts like linear models and residuals\nYou do not need intermediate or expert familiarity with modeling or ML" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-1", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" + }, + { + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-2", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-2", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#who-are-tidymodels", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#who-are-tidymodels", - "title": "1 - Introduction", - "section": "Who are tidymodels?", - "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-2", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-2", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(taxi_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-2", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-2", - "title": "1 - Introduction", - "section": "👀", - "text": "👀" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-3", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-3", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWhat is in this?\n\ntaxi_folds <- vfold_cv(taxi_train)\ntaxi_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <7200/800/8000>\n\n\nTalk about a list column, storing non-atomic types in dataframe" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-3", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-3", - "title": "1 - Introduction", - "section": "👀", - "text": "👀\n\n\nhttps://workshops.tidymodels.org" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-4", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-4", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(taxi_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [6400/1600]> Fold1\n#> 2 <split [6400/1600]> Fold2\n#> 3 <split [6400/1600]> Fold3\n#> 4 <split [6400/1600]> Fold4\n#> 5 <split [6400/1600]> Fold5" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#plan-for-this-workshop", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#plan-for-this-workshop", - "title": "1 - Introduction", - "section": "Plan for this workshop", - "text": "Plan for this workshop\n\nYour data budget\nWhat makes a model\nEvaluating models\nTuning models" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-5", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-5", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(taxi_train, strata = tip)\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10\n\n\nStratification often helps, with very little downside" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-4", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#section-4", - "title": "1 - Introduction", - "section": "", - "text": "Introduce yourself to your neighbors 👋" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-6", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#cross-validation-6", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\ntaxi_folds <- vfold_cv(taxi_train, v = 10, strata = tip)\ntaxi_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [7200/800]> Fold01\n#> 2 <split [7200/800]> Fold02\n#> 3 <split [7200/800]> Fold03\n#> 4 <split [7200/800]> Fold04\n#> 5 <split [7200/800]> Fold05\n#> 6 <split [7200/800]> Fold06\n#> 7 <split [7200/800]> Fold07\n#> 8 <split [7200/800]> Fold08\n#> 9 <split [7200/800]> Fold09\n#> 10 <split [7200/800]> Fold10\n\n\nSet the seed when creating resamples" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nhttps://xkcd.com/1838/" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", + "title": "4 - Evaluating models", + "section": "Fit our model to the resamples", + "text": "Fit our model to the resamples\n\ntaxi_res <- fit_resamples(taxi_wflow, taxi_folds)\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]>\n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]>" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-1", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-1", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\ntaxi_res %>%\n collect_metrics()\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.915 10 0.00309 Preprocessor1_Model1\n#> 2 roc_auc binary 0.624 10 0.0105 Preprocessor1_Model1\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nWe can reliably measure performance using only the training data 🎉" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-2024-edition", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-2024-edition", - "title": "1 - Introduction", - "section": "What is machine learning? (2024 edition)", - "text": "What is machine learning? (2024 edition)\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/\n\n\nIn the early 2010s, “Artificial intelligence” (AI) was largely synonymous with what we’ll refer to as “machine learning” in this workshop. In the late 2010s and early 2020s, AI usually referred to deep learning methods. Since the release of ChatGPT in late 2022, “AI” has come to also encompass large language models / generative models." + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#comparing-metrics", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#comparing-metrics", + "title": "4 - Evaluating models", + "section": "Comparing metrics ", + "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\ntaxi_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 2 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.915 10\n#> 2 roc_auc 0.624 10\n\n\nThe ROC AUC previously was\n\n0.69 for the training set\n0.64 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-2", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-machine-learning-2", - "title": "1 - Introduction", - "section": "What is machine learning?", - "text": "What is machine learning?\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-1", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_taxi <- control_resamples(save_pred = TRUE)\ntaxi_res <- fit_resamples(taxi_wflow, taxi_folds, control = ctrl_taxi)\n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#your-turn", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#your-turn", - "title": "1 - Introduction", - "section": "Your turn", - "text": "Your turn\n\n\nHow are statistics and machine learning related?\nHow are they similar? Different?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-2", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-2", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\ntaxi_preds <- collect_predictions(taxi_res)\ntaxi_preds\n#> # A tibble: 8,000 × 7\n#> id .pred_yes .pred_no .row .pred_class tip .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 Fold01 0.938 0.0615 14 yes yes Preprocessor1_Model1\n#> 2 Fold01 0.946 0.0544 19 yes yes Preprocessor1_Model1\n#> 3 Fold01 0.973 0.0269 33 yes yes Preprocessor1_Model1\n#> 4 Fold01 0.903 0.0971 43 yes yes Preprocessor1_Model1\n#> 5 Fold01 0.973 0.0269 74 yes yes Preprocessor1_Model1\n#> 6 Fold01 0.903 0.0971 103 yes yes Preprocessor1_Model1\n#> 7 Fold01 0.915 0.0851 104 yes no Preprocessor1_Model1\n#> 8 Fold01 0.903 0.0971 124 yes yes Preprocessor1_Model1\n#> 9 Fold01 0.667 0.333 126 yes yes Preprocessor1_Model1\n#> 10 Fold01 0.949 0.0510 128 yes yes Preprocessor1_Model1\n#> # ℹ 7,990 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-tidymodels", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#what-is-tidymodels", - "title": "1 - Introduction", - "section": "What is tidymodels? ", - "text": "What is tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.3.0 ──\n#> ✔ broom 1.0.9 ✔ rsample 1.3.1 \n#> ✔ dials 1.4.1 ✔ tibble 3.3.0 \n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1 \n#> ✔ infer 1.0.9 ✔ tune 1.3.0 \n#> ✔ modeldata 1.5.1 ✔ workflows 1.2.0.9002\n#> ✔ parsnip 1.3.2.9000 ✔ workflowsets 1.1.1 \n#> ✔ purrr 1.1.0 ✔ yardstick 1.3.2 \n#> ✔ recipes 1.3.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-3", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-3", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\ntaxi_preds %>% \n group_by(id) %>%\n taxi_metrics(truth = tip, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.905\n#> 2 Fold02 accuracy binary 0.925\n#> 3 Fold03 accuracy binary 0.926\n#> 4 Fold04 accuracy binary 0.915\n#> 5 Fold05 accuracy binary 0.902\n#> 6 Fold06 accuracy binary 0.912\n#> 7 Fold07 accuracy binary 0.906\n#> 8 Fold08 accuracy binary 0.91 \n#> 9 Fold09 accuracy binary 0.918\n#> 10 Fold10 accuracy binary 0.931\n#> # ℹ 20 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\nRoadmap for today\nMinimal version of predictive modeling process\nFeature engineering and tuning as iterative extensions" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#where-are-the-fitted-models", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#where-are-the-fitted-models", + "title": "4 - Evaluating models", + "section": "Where are the fitted models? ", + "text": "Where are the fitted models? \n\ntaxi_res\n#> # Resampling results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [7200/800]> Fold01 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [7200/800]> Fold02 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [7200/800]> Fold03 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [7200/800]> Fold04 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [7200/800]> Fold05 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [7200/800]> Fold06 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [7200/800]> Fold07 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [7200/800]> Fold08 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [7200/800]> Fold09 <tibble [2 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [7200/800]> Fold10 <tibble [2 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-1", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-1", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#bootstrapping", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#bootstrapping", + "title": "4 - Evaluating models", + "section": "Bootstrapping", + "text": "Bootstrapping" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-2", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-2", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game\n\n\nStress that we are not fitting a model on the entire training set other than for illustrative purposes in deck 2." + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#bootstrapping-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#bootstrapping-1", + "title": "4 - Evaluating models", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(taxi_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [8000/2902]> Bootstrap01\n#> 2 <split [8000/2916]> Bootstrap02\n#> 3 <split [8000/3004]> Bootstrap03\n#> 4 <split [8000/2979]> Bootstrap04\n#> 5 <split [8000/2961]> Bootstrap05\n#> 6 <split [8000/2962]> Bootstrap06\n#> 7 <split [8000/3026]> Bootstrap07\n#> 8 <split [8000/2926]> Bootstrap08\n#> 9 <split [8000/2972]> Bootstrap09\n#> 10 <split [8000/2972]> Bootstrap10\n#> # ℹ 15 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-3", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-3", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-4", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-4", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-3", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-3", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the functions)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-5", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-5", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#monte-carlo-cross-validation", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#monte-carlo-cross-validation", + "title": "4 - Evaluating models", + "section": "Monte Carlo Cross-Validation ", + "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(taxi_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6000/2000]> Resample01\n#> 2 <split [6000/2000]> Resample02\n#> 3 <split [6000/2000]> Resample03\n#> 4 <split [6000/2000]> Resample04\n#> 5 <split [6000/2000]> Resample05\n#> 6 <split [6000/2000]> Resample06\n#> 7 <split [6000/2000]> Resample07\n#> 8 <split [6000/2000]> Resample08\n#> 9 <split [6000/2000]> Resample09\n#> 10 <split [6000/2000]> Resample10" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-6", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-6", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#validation-set", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#validation-set", + "title": "4 - Evaluating models", + "section": "Validation set ", + "text": "Validation set \n\nset.seed(853)\ntaxi_val_split <- initial_validation_split(taxi, strata = tip)\nvalidation_set(taxi_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [6000/2000]> validation\n\n\nA validation set is just another type of resample" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-7", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#the-whole-game-7", - "title": "1 - Introduction", - "section": "The whole game", - "text": "The whole game" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#random-forest-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#random-forest-1", + "title": "4 - Evaluating models", + "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this in Advanced tidymodels!), as long as there are enough trees" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#lets-install-some-packages", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#lets-install-some-packages", - "title": "1 - Introduction", - "section": "Let’s install some packages", - "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"lightgbm\", \"lme4\", \"pak\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\npak::pak(\"simonpcouch/forested\")" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2025-08-nyr/slides/intro-01-introduction.html#our-versions", - "href": "archive/2025-08-nyr/slides/intro-01-introduction.html#our-versions", - "title": "1 - Introduction", - "section": "Our versions", - "text": "Our versions\nR version 4.5.0 (2025-04-11), Quarto (1.6.42)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.4.0\n\n\nbroom\n1.0.9\n\n\nCubist\n0.5.0\n\n\ndials\n1.4.1\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.4\n\n\nembed\n1.1.5.9000\n\n\nfinetune\n1.2.1.9000\n\n\nforested\n0.2.0\n\n\nFormula\n1.2-5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nggplot2\n3.5.2\n\n\nlattice\n0.22-7\n\n\nlightgbm\n4.6.0\n\n\nlme4\n1.1-37\n\n\nmodeldata\n1.5.1\n\n\nparallelly\n1.45.1\n\n\nparsnip\n1.3.2.9000\n\n\nplotmo\n3.6.4\n\n\nplotrix\n3.8-4\n\n\nplumber\n1.3.0\n\n\nprobably\n1.1.0\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\npurrr\n1.1.0\n\n\nranger\n0.17.0\n\n\nrecipes\n1.3.1\n\n\nrpart\n4.1.24\n\n\nrpart.plot\n3.1.3\n\n\nrsample\n1.3.1\n\n\nrules\n1.0.2\n\n\nscales\n1.4.0\n\n\nsplines2\n0.5.4\n\n\nstacks\n1.1.1.9000\n\n\ntext2vec\n0.6.4\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntextrecipes\n1.1.0.9000\n\n\ntibble\n3.3.0\n\n\ntidymodels\n1.3.0\n\n\ntidyr\n1.3.1\n\n\ntune\n1.3.0\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.2.0.9002\n\n\nworkflowsets\n1.1.1\n\n\nyardstick\n1.3.2" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model-1", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_wflow <- workflow(tip ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn", - "title": "3 - What makes a model?", + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-4", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#your-turn-4", + "title": "4 - Evaluating models", "section": "Your turn", - "text": "Your turn\n\nHow do you fit a linear model in R?\nHow many different ways can you think of?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm for linear model\nglmnet for regularized regression\nkeras for regression using TensorFlow\nstan for Bayesian regression\nspark for large data sets\nbrulee for regression using torch" + "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-4", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-4", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nctrl_taxi <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, taxi_folds, control = ctrl_taxi)\ncollect_metrics(rf_res)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.923 10 0.00317 Preprocessor1_Model1\n#> 2 roc_auc binary 0.616 10 0.0147 Preprocessor1_Model1" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-1", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-1", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update-1", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-2", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-2", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-final-fit", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-final-fit", + "title": "4 - Evaluating models", + "section": "The final fit ", + "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# taxi_split has train + test info\nfinal_fit <- last_fit(rf_wflow, taxi_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [8000/2000]> train/test split <tibble> <tibble> <tibble> <workflow>" + }, + { + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.914 Preprocessor1_Model1\n#> 2 roc_auc binary 0.638 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-3", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-3", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() |>\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-1", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-1", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 2,000 × 7\n#> id .pred_yes .pred_no .row .pred_class tip .config \n#> <chr> <dbl> <dbl> <int> <fct> <fct> <chr> \n#> 1 train/test split 0.957 0.0426 4 yes yes Preprocessor1_Mo…\n#> 2 train/test split 0.938 0.0621 10 yes yes Preprocessor1_Mo…\n#> 3 train/test split 0.958 0.0416 19 yes yes Preprocessor1_Mo…\n#> 4 train/test split 0.894 0.106 23 yes yes Preprocessor1_Mo…\n#> 5 train/test split 0.943 0.0573 28 yes yes Preprocessor1_Mo…\n#> 6 train/test split 0.979 0.0213 34 yes yes Preprocessor1_Mo…\n#> 7 train/test split 0.954 0.0463 35 yes yes Preprocessor1_Mo…\n#> 8 train/test split 0.928 0.0722 38 yes yes Preprocessor1_Mo…\n#> 9 train/test split 0.985 0.0147 40 yes yes Preprocessor1_Mo…\n#> 10 train/test split 0.948 0.0523 42 yes no Preprocessor1_Mo…\n#> # ℹ 1,990 more rows" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-4", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-4", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\nlogistic_reg() |>\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-2", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-2", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> tip ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 8000 \n#> Number of independent variables: 6 \n#> Mtry: 2 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.07069778\n\n\nUse this for prediction on new data, like for deploying" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-5", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-5", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game", + "href": "archive/2023-09-posit-conf/intro-04-evaluating-models.html#the-whole-game", + "title": "4 - Evaluating models", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-6", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-6", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" + "objectID": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#your-turn", + "href": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#your-turn", + "title": "6 - Wrapping up", + "section": "Your turn", + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-7", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-7", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\ndecision_tree() |> \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/" + "objectID": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#resources-to-keep-learning", + "href": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#resources-to-keep-learning", + "title": "6 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-8", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#to-specify-a-model-8", - "title": "3 - What makes a model?", - "section": "To specify a model ", - "text": "To specify a model \n\n\n\nChoose a model\nSpecify an engine\nSet the mode" + "objectID": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#feedback", + "href": "archive/2023-09-posit-conf/intro-06-wrapping-up.html#feedback", + "title": "6 - Wrapping up", + "section": "Feedback", + "text": "Feedback\n\nPlease go to http://pos.it/conf-workshop-survey. Your feedback is crucial!\nData from the survey informs curriculum and format decisions for future conf workshops, and we really appreciate you taking the time to provide it." }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-1", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-1", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_spec chunk in your .qmd.\nEdit this code to use a logistic regression model.\n\nAll available models are listed at https://www.tidymodels.org/find/parsnip/\n\n\nExtension/Challenge: Edit this code to use a different model. For example, try using a conditional inference tree as implemented in the partykit package by changing the engine - or try an entirely different model type!\n\n\n\n−+\n05:00" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#the-raw-taxi-data-set", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#the-raw-taxi-data-set", + "title": "Extras - Time-based data splitting", + "section": "The raw taxi data set", + "text": "The raw taxi data set\nWe prepared the data set specifically for this introductory workshop.\nIt looked similar to this:\n\nglimpse(taxi_raw)\n#> Rows: 10,000\n#> Columns: 24\n#> $ trip_id <chr> \"3ac8d4412642a35e9b9a493285814d7983d5a159\",…\n#> $ taxi_id <chr> \"391317d70c5d06deec744062c4595dc1958b200fda…\n#> $ trip_start_timestamp <dttm> 2023-06-10 18:45:00, 2023-05-21 21:30:00, …\n#> $ trip_end_timestamp <dttm> 2023-06-10 19:30:00, 2023-05-21 21:45:00, …\n#> $ trip_seconds <dbl> 3258, 839, 476, 2220, 1588, 2270, 1575, 267…\n#> $ trip_miles <dbl> 17.02, 2.16, 1.05, 17.40, 17.62, 16.36, 18.…\n#> $ pickup_census_tract <dbl> 17031980000, 17031839100, 17031320100, 1703…\n#> $ dropoff_census_tract <dbl> 17031081403, 17031081300, 17031081403, 1703…\n#> $ pickup_community_area <dbl> 76, 32, 32, 76, 32, 76, 76, 32, 32, 33, 8, …\n#> $ dropoff_community_area <dbl> 8, 8, 8, 32, 76, 8, 32, 76, 28, 32, 7, 33, …\n#> $ fare <dbl> 44.50, 10.00, 6.75, 45.00, 44.25, 41.25, 44…\n#> $ tips <dbl> 12.25, 4.00, 2.00, 9.90, 8.00, 9.15, 12.31,…\n#> $ tolls <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ extras <dbl> 4.0, 1.0, 0.0, 4.0, 2.0, 4.0, 4.0, 0.0, 0.0…\n#> $ trip_total <dbl> 61.25, 15.50, 9.25, 58.90, 54.75, 54.90, 61…\n#> $ payment_type <chr> \"Credit Card\", \"Credit Card\", \"Credit Card\"…\n#> $ company <chr> \"Taxicab Insurance Agency Llc\", \"Chicago In…\n#> $ pickup_centroid_latitude <dbl> 41.97907, 41.88099, 41.88499, 41.97907, 41.…\n#> $ pickup_centroid_longitude <dbl> -87.90304, -87.63275, -87.62099, -87.90304,…\n#> $ pickup_centroid_location <chr> \"POINT (-87.9030396611 41.9790708201)\", \"PO…\n#> $ dropoff_centroid_latitude <dbl> 41.89092, 41.89833, 41.89092, 41.87102, 41.…\n#> $ dropoff_centroid_longitude <dbl> -87.61887, -87.62076, -87.61887, -87.63141,…\n#> $ dropoff_centroid_location <chr> \"POINT (-87.6188683546 41.8909220259)\", \"PO…\n#> $ tip <fct> yes, yes, yes, yes, yes, yes, yes, yes, yes…\n\n\ntrip_start_time has date and time rounded to “quarters of the hour”." }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#models-well-be-using-today", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#models-well-be-using-today", - "title": "3 - What makes a model?", - "section": "Models we’ll be using today", - "text": "Models we’ll be using today\n\nLogistic regression\nDecision trees" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#time-nature-of-the-data", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#time-nature-of-the-data", + "title": "Extras - Time-based data splitting", + "section": "Time nature of the data", + "text": "Time nature of the data\nWe assumed only the month, day of the week, and hour mattered and treated each observation as independent.\n\nIf the data have a strong time component, all your data splitting strategies should support the model in estimating temporal trends.\n\n\nThus, don’t sample randomly because this breaks up the time component!" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#splitting-with-time-component", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#splitting-with-time-component", + "title": "Extras - Time-based data splitting", + "section": "Splitting with time component ", + "text": "Splitting with time component \nThe more recent observations are assumed to be more similar to new data, so initial_time_split() puts them into the test set.\nThe function assumes that the data are already ordered.\n\ntaxi_raw <- taxi_raw %>%\n arrange(trip_start_timestamp)\n\ntaxi_split <- initial_time_split(taxi_raw, prop = 3 / 4)\ntaxi_split\n#> <Training/Testing/Total>\n#> <7500/2500/10000>\n\ntaxi_train <- training(taxi_split)\ntaxi_test <- testing(taxi_split)\n\nnrow(taxi_train)\n#> [1] 7500\n \nnrow(taxi_test)\n#> [1] 2500" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression-1", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression-1", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#time-series-resampling", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#time-series-resampling", + "title": "Extras - Time-based data splitting", + "section": "Time series resampling", + "text": "Time series resampling\nThe same idea also applies to resampling: the newer observations go into the assessment set.\nFor example:\n\nFold 1: Take the first X weeks of data as the analysis set, and the next 3 weeks as the assessment set.\nFold 2: Take weeks 2 to X + 1 as the analysis set, and the next 3 weeks as the assessment set.\nand so on" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression-2", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#logistic-regression-2", - "title": "3 - What makes a model?", - "section": "Logistic regression", - "text": "Logistic regression\n\n\n\n\n\n\n\n\n\n\n\n\n\nLogit of outcome probability modeled as linear combination of predictors:\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nFind a sigmoid line that separates the two classes" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#rolling-origin-forecast-resampling", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#rolling-origin-forecast-resampling", + "title": "Extras - Time-based data splitting", + "section": "Rolling origin forecast resampling", + "text": "Rolling origin forecast resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done either way." }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling", + "title": "Extras - Time-based data splitting", + "section": "Times series resampling ", + "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n\n\n\n\n )\n\nUse the trip_start_timestamp column to find the date data." }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees-1", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees-1", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees\n\n\n\n\n\n\n\n\n\n\n\n\n\nSeries of splits or if/then statements based on predictors\nFirst the tree grows until some condition is met (maximum depth, no more data)\nThen the tree is pruned to reduce its complexity" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-1", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-1", + "title": "Extras - Time-based data splitting", + "section": "Times series resampling ", + "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n period = \"week\",\n\n\n\n )\n\nOur units will be in weeks." }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees-2", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#decision-trees-2", - "title": "3 - What makes a model?", - "section": "Decision trees", - "text": "Decision trees" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-2", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-2", + "title": "Extras - Time-based data splitting", + "section": "Times series resampling ", + "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n period = \"week\",\n lookback = 8\n \n \n )\n\nEvery analysis set has 8 weeks of data." }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#all-models-are-wrong-but-some-are-useful", - "title": "3 - What makes a model?", - "section": "All models are wrong, but some are useful!", - "text": "All models are wrong, but some are useful!\n\n\nLogistic regression\n\n\n\n\n\n\n\n\n\n\nDecision trees" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-3", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-3", + "title": "Extras - Time-based data splitting", + "section": "Times series resampling ", + "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n period = \"week\",\n lookback = 8,\n assess_stop = 3,\n\n )\n\nEvery assessment set has 3 weeks of data." }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#workflows-bind-preprocessors-and-models", - "title": "3 - What makes a model?", - "section": "Workflows bind preprocessors and models", - "text": "Workflows bind preprocessors and models\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" + "objectID": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-4", + "href": "archive/2023-09-posit-conf/intro-extra-time-splits.html#times-series-resampling-4", + "title": "Extras - Time-based data splitting", + "section": "Times series resampling ", + "text": "Times series resampling \n\ntaxi_rs <-\n taxi_train %>%\n sliding_period(\n index = \"trip_start_timestamp\", \n period = \"week\",\n lookback = 8,\n assess_stop = 3,\n step = 1\n )\n\nIncrement by 1 week\n\n\ntaxi_rs$splits[[1]] %>% assessment() %>% pluck(\"trip_start_timestamp\") %>% range()\n#> [1] \"2023-03-02 05:15:00 UTC\" \"2023-03-22 22:00:00 UTC\"\n\ntaxi_rs$splits[[2]] %>% assessment() %>% pluck(\"trip_start_timestamp\") %>% range()\n#> [1] \"2023-03-09 07:00:00 UTC\" \"2023-03-29 21:15:00 UTC\"\n\ntaxi_rs$splits[[3]] %>% assessment() %>% pluck(\"trip_start_timestamp\") %>% range()\n#> [1] \"2023-03-16 06:30:00 UTC\" \"2023-04-05 23:45:00 UTC\"" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#what-is-wrong-with-this", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#what-is-wrong-with-this", - "title": "3 - What makes a model?", - "section": "What is wrong with this?", - "text": "What is wrong with this?" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#normas-del-taller", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#normas-del-taller", + "title": "1 - Introducción", + "section": "Normas del taller", + "text": "Normas del taller\n\nCódigo de conducta: TODO-ADD-LATER" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#why-a-workflow", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#why-a-workflow", - "title": "3 - What makes a model?", - "section": "Why a workflow()? ", - "text": "Why a workflow()? \n\n\nWorkflows handle new data better than base R tools in terms of new factor levels\n\n\n\n\nYou can use other preprocessors besides formulas (more on feature engineering in Advanced tidymodels!)\n\n\n\n\nThey can help organize your work when working with multiple models\n\n\n\n\nMost importantly, a workflow captures the entire modeling process: fit() and predict() apply to the preprocessing steps in addition to the actual model fit\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#quien-eres", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#quien-eres", + "title": "1 - Introducción", + "section": "Quien eres?", + "text": "Quien eres?\n\nSabe utilizar la “pipa” de magritr (%>%) o R (|>)\nConoce las funciones de dplyr, tidyr y ggplot2\nEntiende conceptos estadísticos básicos\nNo necesitará ser experto en modelaje o aprendizaje automático" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-1", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-1", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\ntree_spec |> \n fit(forested ~ ., data = forested_train) \n#> parsnip model object\n#> \n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#el-equipo-tidymodels", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#el-equipo-tidymodels", + "title": "1 - Introducción", + "section": "El equipo Tidymodels", + "text": "El equipo Tidymodels\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nAgradecimientos especiales para: Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill y Desirée De Leon" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-2", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-2", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\nworkflow() |>\n add_formula(forested ~ .) |>\n add_model(tree_spec) |>\n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#section-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#section-2", + "title": "1 - Introducción", + "section": "👀", + "text": "👀" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-3", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#a-model-workflow-3", - "title": "3 - What makes a model?", - "section": "A model workflow ", - "text": "A model workflow \n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\nworkflow(forested ~ ., tree_spec) |> \n fit(data = forested_train) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8749 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8749 2427 Yes (0.7225969 0.2774031) \n#> 2) county=Appling,Atkinson,Bacon,Baldwin,Ben Hill,Brantley,Brooks,Bryan,Bulloch,Burke,Butts,Camden,Candler,Carroll,Charlton,Chattahoochee,Chattooga,Cherokee,Clinch,Coffee,Coweta,Crawford,Dade,Dawson,Dodge,Dougherty,Douglas,Echols,Effingham,Elbert,Emanuel,Evans,Fannin,Floyd,Gilmer,Glascock,Greene,Habersham,Hancock,Haralson,Harris,Heard,Jasper,Jeff Davis,Jefferson,Jenkins,Johnson,Jones,Lamar,Lanier,Laurens,Lee,Lincoln,Long,Lumpkin,Marion,McDuffie,Meriwether,Monroe,Montgomery,Morgan,Murray,Oconee,Oglethorpe,Paulding,Pickens,Pierce,Pike,Polk,Putnam,Quitman,Rabun,Randolph,Schley,Screven,Spalding,Stephens,Stewart,Talbot,Taliaferro,Tattnall,Taylor,Telfair,Terrell,Towns,Treutlen,Troup,Twiggs,Union,Upson,Walker,Ware,Warren,Washington,Wayne,Webster,Wheeler,White,Wilcox,Wilkes,Wilkinson 5598 1005 Yes (0.8204716 0.1795284) *\n#> 3) county=Baker,Banks,Barrow,Bartow,Berrien,Bibb,Bleckley,Calhoun,Catoosa,Chatham,Clarke,Clay,Clayton,Cobb,Colquitt,Columbia,Cook,Crisp,Decatur,DeKalb,Dooly,Early,Fayette,Forsyth,Franklin,Fulton,Glynn,Gordon,Grady,Gwinnett,Hall,Hart,Henry,Houston,Irwin,Jackson,Liberty,Lowndes,Macon,Madison,McIntosh,Miller,Mitchell,Muscogee,Newton,Peach,Pulaski,Richmond,Rockdale,Seminole,Sumter,Thomas,Tift,Toombs,Turner,Walton,Whitfield,Worth 3151 1422 Yes (0.5487147 0.4512853) \n#> 6) canopy_cover>=41.5 1773 603 Yes (0.6598985 0.3401015) *\n#> 7) canopy_cover< 41.5 1378 559 No (0.4056604 0.5943396) *" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-planeamos-hacer-en-este-taller", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-planeamos-hacer-en-este-taller", + "title": "1 - Introducción", + "section": "Que planeamos hacer en este taller", + "text": "Que planeamos hacer en este taller\n\nTu presupuesto de datos\nLas partes de un modelo\nEvaluar modelos\nAfinar modelos\n\n\nThis workshop will well-prepare folks going on to the Advanced tidymodels workshop, which will cover feature engineering and much more on hyperparameter tuning." }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-2", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-2", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun the tree_wflow chunk in your .qmd.\nEdit this code to make a workflow with your own model of choice.\n\nExtension/Challenge: Other than formulas, what kinds of preprocessors are supported?\n\n\n\n−+\n05:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#section-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#section-3", + "title": "1 - Introducción", + "section": "", + "text": "Salude a sus vecinos 👋" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#predict-with-your-model", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#predict-with-your-model", - "title": "3 - What makes a model?", - "section": "Predict with your model ", - "text": "Predict with your model \nHow do you use your new tree_fit model?\n\ntree_spec <-\n decision_tree() |> \n set_mode(\"classification\")\n\ntree_fit <-\n workflow(forested ~ ., tree_spec) |> \n fit(data = forested_train)" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático", + "title": "1 - Introducción", + "section": "¿Que es aprendizaje automático?", + "text": "¿Que es aprendizaje automático?" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-3", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-3", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\npredict(tree_fit, new_data = forested_test)\nWhat do you notice about the structure of the result?\n\n\n\n−+\n03:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-1", + "title": "1 - Introducción", + "section": "¿Que es aprendizaje automático?", + "text": "¿Que es aprendizaje automático?\n\n\n¿Este es tu sistema para aprendizaje automático?\nSí, le tiramos los datos a este monton de algebra linear, y después tomamos las repuestas que salen\n¿Y si las respuestas están equivocadas?\nLo empezamos a mezclar hasta que algo se ve como correcto\n\n\nhttps://xkcd.com/1838/" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-4", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-4", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\nRun:\naugment(tree_fit, new_data = forested_test)\nHow does the output compare to the output from predict()?\n\n\n\n−+\n03:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-2", + "title": "1 - Introducción", + "section": "¿Que es aprendizaje automático?", + "text": "¿Que es aprendizaje automático?\n\n\nhttps://vas3k.com/blog/machine_learning/" + }, + { + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-aprendizaje-automático-3", + "title": "1 - Introducción", + "section": "¿Que es aprendizaje automático?", + "text": "¿Que es aprendizaje automático?\n\n\n\n\n\nflowchart TB\n au[Aprendizaje\\nAutomático\\nClásico]\n sp[Supervisada]\n au--Datos numéricos o categóricos-->sp\n us[No supervisada]\n au--Datos no están clasificados-->us\n cl[Clasificación]\n sp--Predice categoría-->cl\n rs[Regresión]\n sp--Predice numero-->rs\n ag[Agrupación]\n us--Divide por similitudes-->ag\n rd[Reducción\\n de dimensiones]\n us--Busca dependencias\\nescondidas-->rd\n as[Asociación]\n us--Identifica secuencias-->as\n \n\n\n\n\n\n\n\nhttps://vas3k.com/blog/machine_learning/" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#tu-turno", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#tu-turno", + "title": "1 - Introducción", + "section": "Tu turno", + "text": "Tu turno\n\n\n¿Como se relacionan las estadísticas y el aprendizaje automático?\n¿Como se parecen? ¿Cuales son sus diferencias?\n\n\n\n−+\n03:00\n\n\n\n\nthe “two cultures”\nmodel first vs. data first\ninference vs. prediction" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model-1", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model-1", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nlibrary(rpart.plot)\ntree_fit |>\n extract_fit_engine() |>\n rpart.plot(roundint = FALSE)\n\nYou can extract_*() several components of your fitted workflow.\n\n⚠️ Never predict() with any extracted components!\n\nroundint = FALSE is only to quiet a warning" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-tidymodels", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#que-es-tidymodels", + "title": "1 - Introducción", + "section": "¿Que es Tidymodels? ", + "text": "¿Que es Tidymodels? \n\nlibrary(tidymodels)\n#> ── Attaching packages ──────────────────────────── tidymodels 1.1.1 ──\n#> ✔ broom 1.0.5 ✔ rsample 1.2.0 \n#> ✔ dials 1.2.1 ✔ tibble 3.2.1 \n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1 \n#> ✔ infer 1.0.6 ✔ tune 1.1.2 \n#> ✔ modeldata 1.3.0 ✔ workflows 1.1.4 \n#> ✔ parsnip 1.2.0 ✔ workflowsets 1.0.1 \n#> ✔ purrr 1.0.2 ✔ yardstick 1.3.0 \n#> ✔ recipes 1.0.10\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n#> • Learn how to get started at https://www.tidymodels.org/start/" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model-2", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#understand-your-model-2", - "title": "3 - What makes a model?", - "section": "Understand your model ", - "text": "Understand your model \nHow do you understand your new tree_fit model?\n\nYou can use your fitted workflow for model and/or prediction explanations:\n\n\n\noverall variable importance, such as with the vip package\n\n\n\n\nflexible model explainers, such as with the DALEXtra package\n\n\n\nLearn more at https://www.tmwr.org/explain.html" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller", + "title": "1 - Introducción", + "section": "Expectativas del taller", + "text": "Expectativas del taller\n\nEl “mapa” de hoy\nProcesos básicos de los modelos predictivos\nVer la ingenieria de caraterísticas (feature engineering) y afinamiento como extenciones cíclicas" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-5", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#your-turn-5", - "title": "3 - What makes a model?", - "section": "Your turn", - "text": "Your turn\n\n\nExtract the model engine object from your fitted workflow and check it out.\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-1", + "title": "1 - Introducción", + "section": "Expectativas del taller", + "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts" }, { - "objectID": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#the-whole-game---status-update", - "href": "archive/2025-08-nyr/slides/intro-03-what-makes-a-model.html#the-whole-game---status-update", - "title": "3 - What makes a model?", - "section": "The whole game - status update", - "text": "The whole game - status update\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-2", + "title": "1 - Introducción", + "section": "Expectativas del taller", + "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n tr --> dt" }, { - "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#tuning-parameters", - "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#tuning-parameters", - "title": "5 - Tuning models", - "section": "Tuning parameters", - "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-3", + "title": "1 - Introducción", + "section": "Expectativas del taller", + "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n tr --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n tr --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n tr --> rf" }, { - "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#optimize-tuning-parameters", - "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#optimize-tuning-parameters", - "title": "5 - Tuning models", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-4", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-4", + "title": "1 - Introducción", + "section": "Expectativas del taller", + "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf" }, { - "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#optimize-tuning-parameters-1", - "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#optimize-tuning-parameters-1", - "title": "5 - Tuning models", - "section": "Optimize tuning parameters", - "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-5", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-5", + "title": "1 - Introducción", + "section": "Expectativas del taller", + "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm" }, { - "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#specifying-tuning-parameters", - "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#specifying-tuning-parameters", - "title": "5 - Tuning models", - "section": "Specifying tuning parameters", - "text": "Specifying tuning parameters\nLet’s take our previous random forest workflow and tag for tuning the minimum number of data points in each node:\n\nrf_spec <- rand_forest(min_n = tune()) |> \n set_mode(\"classification\")\n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-6", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-6", + "title": "1 - Introducción", + "section": "Expectativas del taller", + "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#FBE9BF,stroke:#666,color:#000\n sm --> fm\n tr --> fm" }, { - "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#try-out-multiple-values", - "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#try-out-multiple-values", - "title": "5 - Tuning models", - "section": "Try out multiple values", - "text": "Try out multiple values\ntune_grid() works similar to fit_resamples() but covers multiple parameter values:\n\nset.seed(22)\nrf_res <- tune_grid(\n rf_wflow,\n forested_folds,\n grid = 5\n)" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-7", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#expectativas-del-taller-7", + "title": "1 - Introducción", + "section": "Expectativas del taller", + "text": "Expectativas del taller\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#FDF4E3,stroke:#666,color:#000\n tr --> rs\n lg[Regresión\\nlogística]\n style lg fill:#FDF4E3,stroke:#666,color:#000\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#FDF4E3,stroke:#666,color:#000\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#FDF4E3,stroke:#666,color:#000\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#FBE9BF,stroke:#666,color:#000\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#E5E7FD,stroke:#666,color:#000\n fm --> vm\n ts --> vm" }, { - "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#compare-results", - "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#compare-results", - "title": "5 - Tuning models", - "section": "Compare results", - "text": "Compare results\nInspecting results and selecting the best-performing hyperparameter(s):\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 40 roc_auc binary 0.762 10 0.0107 Preprocessor1_Model1\n#> 2 31 roc_auc binary 0.761 10 0.0101 Preprocessor1_Model2\n#> 3 22 roc_auc binary 0.760 10 0.0103 Preprocessor1_Model5\n#> 4 12 roc_auc binary 0.758 10 0.0101 Preprocessor1_Model4\n#> 5 3 roc_auc binary 0.755 10 0.00969 Preprocessor1_Model3\n\nbest_parameter <- select_best(rf_res)\nbest_parameter\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 40 Preprocessor1_Model1\n\ncollect_metrics() and autoplot() are also available." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#instalemos-unos-paquetes", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#instalemos-unos-paquetes", + "title": "1 - Introducción", + "section": "Instalemos unos paquetes", + "text": "Instalemos unos paquetes\n\npkgs <- \n c(\"bonsai\", \"doParallel\", \"embed\", \"finetune\", \"lightgbm\", \"lme4\",\n \"plumber\", \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\",\n \"splines2\", \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \n \"vetiver\", \"remotes\")\n\ninstall.packages(pkgs)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#the-final-fit", - "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#the-final-fit", - "title": "5 - Tuning models", - "section": "The final fit", - "text": "The final fit\n\nrf_wflow <- finalize_workflow(rf_wflow, best_parameter)\n\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.764 Preprocessor1_Model1\n#> 2 roc_auc binary 0.764 Preprocessor1_Model1\n#> 3 brier_class binary 0.161 Preprocessor1_Model1" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#nuestras-versiones", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-01-introduccion.html#nuestras-versiones", + "title": "1 - Introducción", + "section": "Nuestras versiones", + "text": "Nuestras versiones\nR version 4.3.2 (2023-10-31), Quarto (1.4.550)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.2.1\n\n\nbroom\n1.0.5\n\n\ndials\n1.2.1\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nembed\n1.1.3\n\n\nfinetune\n1.1.0\n\n\nggplot2\n3.5.0\n\n\nlightgbm\n4.3.0\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nlme4\n1.1-35.1\n\n\nmodeldata\n1.3.0\n\n\nparsnip\n1.2.0\n\n\nplumber\n1.2.1\n\n\nprobably\n1.0.3\n\n\npurrr\n1.0.2\n\n\nranger\n0.16.0\n\n\nrecipes\n1.0.10\n\n\nremotes\n2.4.2.1\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nrpart\n4.1.23\n\n\nrpart.plot\n3.1.2\n\n\nrsample\n1.2.0\n\n\nrules\n1.0.2\n\n\nscales\n1.3.0\n\n\nsplines2\n0.5.1\n\n\nstacks\n1.0.3\n\n\ntext2vec\n0.6.4\n\n\ntextrecipes\n1.0.6\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.1.1\n\n\ntidyr\n1.3.1\n\n\ntune\n1.1.2\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.1.4\n\n\nworkflowsets\n1.0.1\n\n\nyardstick\n1.3.0" }, { - "objectID": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#your-turn", - "href": "archive/2025-08-nyr/slides/intro-05-tuning-models.html#your-turn", - "title": "5 - Tuning models", - "section": "Your turn", - "text": "Your turn\n\nModify your model workflow to tune one or more parameters.\nUse grid search to find the best parameter(s).\n\n\n\n−+\n05:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno", + "title": "3 - Las partes de un modelo", + "section": "Tu turno", + "text": "Tu turno\n\n¿Como ajustar un modelo linear en R?\n¿De cuantas maneras sabes como ajustar este tipo de modelo?\n\n\n\n−+\n03:00\n\n\n\n\n\nlm para regresión linear\nglm para regresión linear generalizada\nglmnet para regresión regularizada\nkeras para regresión dentro de Tensorflow\nstan para regresión bayensiana\nspark para datos “grandes”" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#looking-at-the-predictors", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#looking-at-the-predictors", - "title": "Extras - Recipes", - "section": "Looking at the predictors", - "text": "Looking at the predictors\n\nforested_train\n#> # A tibble: 8,749 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 1997 66 82 10 Tree 12.2 1315 18.4 1.88 24.9 11.8 121 1888 66 -84.9 32.4\n#> 2 No 1997 284 -99 58 Tree 10.3 1236 16.1 -0.26 22.3 9.92 68 1586 80 -85.0 34.1\n#> 3 Yes 2022 130 86 15 Tree 11.8 1194 17.6 1.3 24.1 11.2 61 1753 96 -83.0 33.1\n#> 4 Yes 2021 202 -55 3 Tree 10.7 1235 16.6 0.05 23.0 10.2 72 1682 65 -83.0 33.9\n#> 5 Yes 1995 75 -89 1 Tree 13.8 1256 19.2 3.63 25.5 12.9 57 1796 88 -83.4 31.3\n#> 6 No 1995 110 -53 5 Tree 12.4 1236 18.6 2.53 24.8 12.4 102 1835 51 -83.9 32.2\n#> 7 Yes 2022 111 73 12 Tree 11.5 1168 17.4 1 24.0 10.9 67 1772 84 -82.2 33.6\n#> 8 Yes 1997 230 96 14 Tree 9.98 1373 15.4 -1.35 21.8 9.03 46 1552 68 -85.3 34.8\n#> 9 Yes 2002 160 -88 13 Tree 11.1 1219 16.9 0.07 23.6 10.2 53 1731 95 -83.2 33.6\n#> 10 Yes 2020 39 9 6 Tree 13.9 1237 19.2 3.25 25.6 12.8 58 1812 86 -82.0 31.7\n#> # ℹ 8,739 more rows\n#> # ℹ 2 more variables: land_type <fct>, county <fct>" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo", + "title": "3 - Las partes de un modelo", + "section": "Para especificar un modelo ", + "text": "Para especificar un modelo \n\n\n\n\nElije un modelo\nEspecifica el “motor”\nEstablece el modo" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#working-with-other-models", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#working-with-other-models", - "title": "Extras - Recipes", - "section": "Working with other models", - "text": "Working with other models\nSome models can’t handle non-numeric data\n\nLinear Regression\nK Nearest Neighbors\n\n\n\nSome models struggle if numeric predictors aren’t scaled\n\nK Nearest Neighbors\nAnything using gradient descent" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-1", + "title": "3 - Las partes de un modelo", + "section": "Para especificar un modelo ", + "text": "Para especificar un modelo \n\nlogistic_reg()\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glm\n\n\nModels have default engines" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#types-of-needed-preprocessing", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#types-of-needed-preprocessing", - "title": "Extras - Recipes", - "section": "Types of needed preprocessing", - "text": "Types of needed preprocessing\n\nDo qualitative predictors require a numeric encoding?\nShould columns with a single unique value be removed?\nDoes the model struggle with missing data?\nDoes the model struggle with correlated predictors?\nShould predictors be centered and scaled?\nIs it helpful to transform predictors to be more symmetric?\n\n\nhttps://www.tmwr.org/pre-proc-table.html" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-2", + "title": "3 - Las partes de un modelo", + "section": "Para especificar un modelo ", + "text": "Para especificar un modelo \n\n\n\nElije un modelo\nEspecifica el “motor”\nEstablece el modo" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-preprocessing", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-preprocessing", - "title": "Extras - Recipes", - "section": "Two types of preprocessing", - "text": "Two types of preprocessing" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-3", + "title": "3 - Las partes de un modelo", + "section": "Para especificar un modelo ", + "text": "Para especificar un modelo \n\nlogistic_reg() %>%\n set_engine(\"glmnet\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: glmnet" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-preprocessing-1", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-preprocessing-1", - "title": "Extras - Recipes", - "section": "Two types of preprocessing", - "text": "Two types of preprocessing" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-4", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-4", + "title": "3 - Las partes de un modelo", + "section": "Para especificar un modelo ", + "text": "Para especificar un modelo \n\nlogistic_reg() %>%\n set_engine(\"stan\")\n#> Logistic Regression Model Specification (classification)\n#> \n#> Computational engine: stan" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#general-definitions", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#general-definitions", - "title": "Extras - Recipes", - "section": "General definitions", - "text": "General definitions\n\nData preprocessing is what you do to make your model successful.\nFeature engineering is what you do to the original predictors to make the model do the least work to perform great." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-5", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-5", + "title": "3 - Las partes de un modelo", + "section": "Para especificar un modelo ", + "text": "Para especificar un modelo \n\n\n\nElije un modelo\nEspecifica el “motor”\nEstablece el modo" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#working-with-dates", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#working-with-dates", - "title": "Extras - Recipes", - "section": "Working with dates", - "text": "Working with dates\nDatetime variables are automatically converted to an integer if given as a raw predictor. To avoid this, it can be re-encoded as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nLeap year\nIndicators for holidays" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-6", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-6", + "title": "3 - Las partes de un modelo", + "section": "Para especificar un modelo ", + "text": "Para especificar un modelo \n\ndecision_tree()\n#> Decision Tree Model Specification (unknown mode)\n#> \n#> Computational engine: rpart\n\n\nSome models have a default mode" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-transformations", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#two-types-of-transformations", - "title": "Extras - Recipes", - "section": "Two types of transformations", - "text": "Two types of transformations\n\n\n\nStatic\n\nSquare root, log, inverse\nDummies for known levels\nDate time extractions\n\n\nTrained\n\nCentering & scaling\nImputation\nPCA\nAnything for unknown factor levels\n\n\n\nTrained methods need to calculate sufficient information to be applied again." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-7", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#para-especificar-un-modelo-7", + "title": "3 - Las partes de un modelo", + "section": "Para especificar un modelo ", + "text": "Para especificar un modelo \n\ndecision_tree() %>% \n set_mode(\"classification\")\n#> Decision Tree Model Specification (classification)\n#> \n#> Computational engine: rpart\n\n\n\n\nEl listado de modelos se encuentra aquí: https://www.tidymodels.org/find/parsnip/" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#the-recipes-package", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#the-recipes-package", - "title": "Extras - Recipes", - "section": "The recipes package", - "text": "The recipes package\n\n\nModular + extensible\nWorks well with pipes ,|> and %>%\nDeferred evaluation\nIsolates test data from training data\nCan do things formulas can’t" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#to-specify-a-model", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#to-specify-a-model", + "title": "3 - Las partes de un modelo", + "section": "To specify a model ", + "text": "To specify a model \n\n\n\nElije un modelo\nEspecifica el “motor”\nEstablece el modo" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-1", + "title": "3 - Las partes de un modelo", + "section": "Tu turno", + "text": "Tu turno\n\nCorre el “chunk” arbol_espec en tu archivo .qmd.\nCambia el código para que utilize una regresión linear\n\nEl listado de modelos se encuentra aquí: https://www.tidymodels.org/find/parsnip/\n\n\nReto: Edita el código para usar otro tipo de modelo. Por ejemplo, trata de usar el árbol disponible dentro del paquete partykit, o trata un modelo de tipo diferente\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-1", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-1", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())\n\n\nStart by calling recipe() to denote the data source and variables used." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#los-modelos-que-usaremos-hoy", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#los-modelos-que-usaremos-hoy", + "title": "3 - Las partes de un modelo", + "section": "Los modelos que usaremos hoy", + "text": "Los modelos que usaremos hoy\n\nRegresión logística\nArboles de decisión" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-2", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-2", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>\n  step_normalize(all_numeric_predictors())\n\n\nSpecify what actions to take by adding step_*()s." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística", + "title": "3 - Las partes de un modelo", + "section": "Regresión logística", + "text": "Regresión logística" + }, + { + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística-1", + "title": "3 - Las partes de un modelo", + "section": "Regresión logística", + "text": "Regresión logística" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-3", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#how-to-write-a-recipe-3", - "title": "Extras - Recipes", - "section": "How to write a recipe", - "text": "How to write a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>   step_normalize(all_numeric_predictors())\n\n\nUse {tidyselect} and recipes-specific selectors to denote affected variables." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#regresión-logística-2", + "title": "3 - Las partes de un modelo", + "section": "Regresión logística", + "text": "Regresión logística\n\n\n\n\n\n\n\n\n\n\n\n\n\nEl logit del la probabilidad del resultado\n\n\\(log(\\frac{p}{1 - p}) = \\beta_0 + \\beta_1\\cdot \\text{A}\\)\n\nCalcular la línea de sigmoideo que separan las dos clases" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe", - "title": "Extras - Recipes", - "section": "Using a recipe", - "text": "Using a recipe\n\nforested_rec <- recipe(forested ~ ., data = forested_train) |>\n  step_dummy(all_nominal_predictors()) |>\n  step_zv(all_predictors()) |>\n  step_log(canopy_cover, offset = 0.5) |>   step_normalize(all_numeric_predictors())\n\n\nSave the recipe we like so that we can use it in various places, e.g., with different models." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión", + "title": "3 - Las partes de un modelo", + "section": "Arboles de decisión", + "text": "Arboles de decisión" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe-with-workflows", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe-with-workflows", - "title": "Extras - Recipes", - "section": "Using a recipe with workflows", - "text": "Using a recipe with workflows\nRecipes are typically combined with a model in a workflow() object:\n\n\nforested_wflow <- workflow() |>\n  add_recipe(forested_rec) |>\n  add_model(linear_reg())" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión-1", + "title": "3 - Las partes de un modelo", + "section": "Arboles de decisión", + "text": "Arboles de decisión\n\n\n\n\n\n\n\n\n\n\n\n\n\nEl resultado es decidido usando una serie de decisiones basados en las variables predictivas\nPrimero, el arbol crece hasta que se llega a una decision\nDespués, el arbol es recortado para que no sea tan complejo" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#recipes-are-estimated", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#recipes-are-estimated", - "title": "Extras - Recipes", - "section": "Recipes are estimated", - "text": "Recipes are estimated\nEvery preprocessing step in a recipe that involved calculations uses the training set. For example:\n\nLevels of a factor\nDetermination of zero-variance\nNormalization\nFeature extraction\n\nOnce a recipe is added to a workflow, this occurs when fit() is called." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#arboles-de-decisión-2", + "title": "3 - Las partes de un modelo", + "section": "Arboles de decisión", + "text": "Arboles de decisión" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#debugging-a-recipe", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#debugging-a-recipe", - "title": "Extras - Recipes", - "section": "Debugging a recipe", - "text": "Debugging a recipe\n\nTypically, you will want to use a workflow to estimate and apply a recipe.\n\n\n\nIf you have an error and need to debug your recipe, the original recipe object (e.g. forested_rec) can be estimated manually with a function called prep(). It is analogous to fit(). See TMwR section 16.4.\n\n\n\n\nAnother function, bake(), is analogous to predict(), and gives you the processed data back." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#todos-los-modelos-están-equivocados-pero-algunos-pueden-ser-utíl", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#todos-los-modelos-están-equivocados-pero-algunos-pueden-ser-utíl", + "title": "3 - Las partes de un modelo", + "section": "Todos los modelos están equivocados, pero algunos pueden ser utíl", + "text": "Todos los modelos están equivocados, pero algunos pueden ser utíl\n\n\nRegresión logística\n\n\n\n\n\n\n\n\n\n\nArboles de decisión" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#your-turn", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#your-turn", - "title": "Extras - Recipes", - "section": "Your turn", - "text": "Your turn\n\n\nTake the recipe and prep() then bake() it to see what the resulting data set looks like.\nTry removing steps to see how the result changes.\n\n\n\n\n−+\n05:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#los-flujos-de-trabajo-combinan-el-preprocesamiento-y-el-modelo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#los-flujos-de-trabajo-combinan-el-preprocesamiento-y-el-modelo", + "title": "3 - Las partes de un modelo", + "section": "Los flujos de trabajo combinan el “preprocesamiento” y el modelo", + "text": "Los flujos de trabajo combinan el “preprocesamiento” y el modelo\n\n\n\n\n\nflowchart LR\n dt[Datos]\n pr[Predictores]\n subgraph fl[Flujo del modelo]\n pca[PCA]\n ls[Menos cuadrados]\n pca-->ls\n end\n dt-->fl\n pr-->fl\n ft[Modelo ajustado]\n fl-->ft\n style fl fill:#fff,stroke:#666,color:#000\n\n\n\n\n\n\n\nExplain that PCA that is a preprocessor / dimensionality reduction, used to decorrelate data" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#printing-a-recipe", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#printing-a-recipe", - "title": "Extras - Recipes", - "section": "Printing a recipe", - "text": "Printing a recipe\n\nforested_rec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Operations\n#> • Dummy variables from: all_nominal_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Log transformation on: canopy_cover\n#> • Centering and scaling for: all_numeric_predictors()" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#que-no-está-bien-en-este-flujo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#que-no-está-bien-en-este-flujo", + "title": "3 - Las partes de un modelo", + "section": "¿Que no está bien en este flujo?", + "text": "¿Que no está bien en este flujo?\n\n\n\n\n\nflowchart LR\n dt[Datos]\n pr[Predictores]\n pca[PCA]\n subgraph fl[Flujo del modelo]\n ls[Menos cuadrados]\n end\n pca-->fl\n dt-->pca\n pr-->pca\n ft[Modelo ajustado]\n fl-->ft\n style fl fill:#fff,stroke:#666,color:#000" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#prepping-a-recipe", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#prepping-a-recipe", - "title": "Extras - Recipes", - "section": "Prepping a recipe", - "text": "Prepping a recipe\n\nprep(forested_rec)\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 18\n#> \n#> ── Training information\n#> Training data contained 8749 data points and no incomplete rows.\n#> \n#> ── Operations\n#> • Dummy variables from: tree_no_tree, land_type, county | Trained\n#> • Zero variance filter removed: <none> | Trained\n#> • Log transformation on: canopy_cover | Trained\n#> • Centering and scaling for: year elevation, ... | Trained" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#por-que-utilizar-workflow", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#por-que-utilizar-workflow", + "title": "3 - Las partes de un modelo", + "section": "¿Por que utilizar workflow()? ", + "text": "¿Por que utilizar workflow()? \n\n\nworkflow maneja nuevos niveles factoriales mejor que las herramientas regulares de R\n\n\n\n\nPuedes utilizar “preprocesadores” que no son formulas\n\n\n\n\nAyudan a organizar un proyecto que utiliza varios modelos\n\n\n\n\nAún más importante, un flujo de workflow encaja todo el processo de modelamiento, desde la creación (fit()) del modelo, hasta el uso del modelo (predict())\n\n\nTwo ways workflows handle levels better than base R:\n\nEnforces that new levels are not allowed at prediction time (this is an optional check that can be turned off)\nRestores missing levels that were present at fit time, but happen to be missing at prediction time (like, if your “new” data just doesn’t have an instance of that level)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#baking-a-recipe", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#baking-a-recipe", - "title": "Extras - Recipes", - "section": "Baking a recipe", - "text": "Baking a recipe\n\nprep(forested_rec) |>\n bake(new_data = forested_train)\n#> # A tibble: 8,749 × 177\n#> year elevation eastness roughness dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat forested\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> \n#> 1 -1.15 -0.521 1.13 0.0463 -0.232 0.178 0.172 -0.107 0.338 -0.00909 3.10 1.01 0.407 -1.44 -0.0184 Yes \n#> 2 -1.15 1.10 -1.50 3.91 -1.39 -0.568 -1.31 -1.33 -1.28 -1.28 0.0408 -0.941 0.652 -1.51 1.36 No \n#> 3 0.892 -0.0440 1.19 0.449 -0.422 -0.965 -0.305 -0.438 -0.170 -0.438 -0.363 0.140 0.884 0.331 0.556 Yes \n#> 4 0.810 0.493 -0.858 -0.518 -1.11 -0.577 -1.01 -1.15 -0.878 -1.10 0.272 -0.320 0.388 0.331 1.26 Yes \n#> 5 -1.31 -0.454 -1.35 -0.679 0.748 -0.379 0.701 0.893 0.683 0.702 -0.594 0.418 0.773 -0.0643 -0.992 Yes \n#> 6 -1.31 -0.193 -0.829 -0.357 -0.0546 -0.568 0.331 0.264 0.281 0.366 2.00 0.670 0.0802 -0.468 -0.237 No \n#> 7 0.892 -0.186 1.00 0.207 -0.631 -1.21 -0.444 -0.609 -0.264 -0.619 -0.0169 0.262 0.714 1.01 0.929 Yes \n#> 8 -1.15 0.702 1.34 0.369 -1.56 0.726 -1.77 -1.95 -1.60 -1.87 -1.23 -1.16 0.445 -1.77 2.03 Yes \n#> 9 -0.740 0.180 -1.34 0.288 -0.858 -0.729 -0.801 -1.14 -0.515 -1.08 -0.825 -0.00273 0.870 0.122 0.988 Yes \n#> 10 0.729 -0.722 0.0719 -0.276 0.816 -0.559 0.708 0.676 0.745 0.635 -0.536 0.521 0.744 1.22 -0.653 Yes \n#> # ℹ 8,739 more rows\n#> # ℹ 161 more variables: tree_no_tree_No.tree <dbl>, land_type_Non.tree.vegetation <dbl>, land_type_Tree <dbl>, county_Atkinson <dbl>, county_Bacon <dbl>, county_Baker <dbl>, county_Baldwin <dbl>,\n#> # county_Banks <dbl>, county_Barrow <dbl>, county_Bartow <dbl>, county_Ben.Hill <dbl>, county_Berrien <dbl>, county_Bibb <dbl>, county_Bleckley <dbl>, county_Brantley <dbl>, county_Brooks <dbl>,\n#> # county_Bryan <dbl>, county_Bulloch <dbl>, county_Burke <dbl>, county_Butts <dbl>, county_Calhoun <dbl>, county_Camden <dbl>, county_Candler <dbl>, county_Carroll <dbl>, county_Catoosa <dbl>,\n#> # county_Charlton <dbl>, county_Chatham <dbl>, county_Chattahoochee <dbl>, county_Chattooga <dbl>, county_Cherokee <dbl>, county_Clarke <dbl>, county_Clay <dbl>, county_Clayton <dbl>,\n#> # county_Clinch <dbl>, county_Cobb <dbl>, county_Coffee <dbl>, county_Colquitt <dbl>, county_Columbia <dbl>, county_Cook <dbl>, county_Coweta <dbl>, county_Crawford <dbl>, county_Crisp <dbl>,\n#> # county_Dade <dbl>, county_Dawson <dbl>, county_Decatur <dbl>, county_DeKalb <dbl>, county_Dodge <dbl>, county_Dooly <dbl>, county_Dougherty <dbl>, county_Douglas <dbl>, county_Early <dbl>, …" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-1", + "title": "3 - Las partes de un modelo", + "section": "El flujo de trabajo de un modelo ", + "text": "El flujo de trabajo de un modelo \n\narbol_espec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\narbol_espec %>% \n fit(propina ~ ., data = taxi_entrenar) \n#> parsnip model object\n#> \n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 si (0.92300000 0.07700000) \n#> 2) distancia>=14.12 2041 68 si (0.96668300 0.03331700) *\n#> 3) distancia< 14.12 5959 548 si (0.90803826 0.09196174) \n#> 6) distancia< 5.275 5419 450 si (0.91695885 0.08304115) *\n#> 7) distancia>=5.275 540 98 si (0.81851852 0.18148148) \n#> 14) compania=Chicago Independents,City Service,otra,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc 478 68 si (0.85774059 0.14225941) *\n#> 15) compania=Flash Cab 62 30 si (0.51612903 0.48387097) \n#> 30) dia=Jue 12 2 si (0.83333333 0.16666667) *\n#> 31) dia=Lun,Mar,Mie,Vie,Sab,Dom 50 22 no (0.44000000 0.56000000) \n#> 62) distancia>=11.77 14 4 si (0.71428571 0.28571429) *\n#> 63) distancia< 11.77 36 12 no (0.33333333 0.66666667) *" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\nOnce a recipe as been estimated, there are various bits of information saved in it.\n\nThe tidy() function can be used to get specific results from the recipe." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-2", + "title": "3 - Las partes de un modelo", + "section": "El flujo de trabajo de un modelo ", + "text": "El flujo de trabajo de un modelo \n\narbol_espec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\nworkflow() %>%\n add_formula(propina ~ .) %>%\n add_model(arbol_espec) %>%\n fit(data = taxi_entrenar) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 si (0.92300000 0.07700000) \n#> 2) distancia>=14.12 2041 68 si (0.96668300 0.03331700) *\n#> 3) distancia< 14.12 5959 548 si (0.90803826 0.09196174) \n#> 6) distancia< 5.275 5419 450 si (0.91695885 0.08304115) *\n#> 7) distancia>=5.275 540 98 si (0.81851852 0.18148148) \n#> 14) compania=Chicago Independents,City Service,otra,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc 478 68 si (0.85774059 0.14225941) *\n#> 15) compania=Flash Cab 62 30 si (0.51612903 0.48387097) \n#> 30) dia=Jue 12 2 si (0.83333333 0.16666667) *\n#> 31) dia=Lun,Mar,Mie,Vie,Sab,Dom 50 22 no (0.44000000 0.56000000) \n#> 62) distancia>=11.77 14 4 si (0.71428571 0.28571429) *\n#> 63) distancia< 11.77 36 12 no (0.33333333 0.66666667) *" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#your-turn-1", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#your-turn-1", - "title": "Extras - Recipes", - "section": "Your turn", - "text": "Your turn\n\nTake a prepped recipe and use the tidy() function on it.\nUse the number argument to inspect different steps.\n\n\n\n\n−+\n05:00" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#el-flujo-de-trabajo-de-un-modelo-3", + "title": "3 - Las partes de un modelo", + "section": "El flujo de trabajo de un modelo ", + "text": "El flujo de trabajo de un modelo \n\narbol_espec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\nworkflow(propina ~ ., arbol_espec) %>% \n fit(data = taxi_entrenar) \n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: decision_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> n= 8000 \n#> \n#> node), split, n, loss, yval, (yprob)\n#> * denotes terminal node\n#> \n#> 1) root 8000 616 si (0.92300000 0.07700000) \n#> 2) distancia>=14.12 2041 68 si (0.96668300 0.03331700) *\n#> 3) distancia< 14.12 5959 548 si (0.90803826 0.09196174) \n#> 6) distancia< 5.275 5419 450 si (0.91695885 0.08304115) *\n#> 7) distancia>=5.275 540 98 si (0.81851852 0.18148148) \n#> 14) compania=Chicago Independents,City Service,otra,Sun Taxi,Taxi Affiliation Services,Taxicab Insurance Agency Llc 478 68 si (0.85774059 0.14225941) *\n#> 15) compania=Flash Cab 62 30 si (0.51612903 0.48387097) \n#> 30) dia=Jue 12 2 si (0.83333333 0.16666667) *\n#> 31) dia=Lun,Mar,Mie,Vie,Sab,Dom 50 22 no (0.44000000 0.56000000) \n#> 62) distancia>=11.77 14 4 si (0.71428571 0.28571429) *\n#> 63) distancia< 11.77 36 12 no (0.33333333 0.66666667) *" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe-1", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe-1", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\n\nprep(forested_rec) |>\n tidy()\n#> # A tibble: 4 × 6\n#> number operation type trained skip id \n#> <int> <chr> <chr> <lgl> <lgl> <chr> \n#> 1 1 step dummy TRUE FALSE dummy_hIEnQ \n#> 2 2 step zv TRUE FALSE zv_ZrQBx \n#> 3 3 step log TRUE FALSE log_6es7X \n#> 4 4 step normalize TRUE FALSE normalize_XsIxb" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-2", + "title": "3 - Las partes de un modelo", + "section": "Tu turno", + "text": "Tu turno\n\nCorre el “chunk” arbol_flujo en tu archivo .qmd.\nEdita el codigo para crear un flujo con un modelo que tú elijas\n\nReto: ¿Que otros “preprocesadores”, aparte de formulas, podemos usar en workflow?\n\n\n\n−+\n05:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe-2", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#tidying-a-recipe-2", - "title": "Extras - Recipes", - "section": "Tidying a recipe", - "text": "Tidying a recipe\n\nprep(forested_rec) |>\n tidy(number = 1)\n#> # A tibble: 161 × 3\n#> terms columns id \n#> <chr> <chr> <chr> \n#> 1 tree_no_tree No tree dummy_hIEnQ\n#> 2 land_type Non-tree vegetation dummy_hIEnQ\n#> 3 land_type Tree dummy_hIEnQ\n#> 4 county Atkinson dummy_hIEnQ\n#> 5 county Bacon dummy_hIEnQ\n#> 6 county Baker dummy_hIEnQ\n#> 7 county Baldwin dummy_hIEnQ\n#> 8 county Banks dummy_hIEnQ\n#> 9 county Barrow dummy_hIEnQ\n#> 10 county Bartow dummy_hIEnQ\n#> # ℹ 151 more rows" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#predecir-usando-tu-modelo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#predecir-usando-tu-modelo", + "title": "3 - Las partes de un modelo", + "section": "Predecir usando tu modelo ", + "text": "Predecir usando tu modelo \n¿Como usar su nuevo modelo arbol_flujo?\n\narbol_espec <-\n decision_tree(cost_complexity = 0.002) %>% \n set_mode(\"classification\")\n\narbol_flujo <-\n workflow(propina ~ ., arbol_espec) %>% \n fit(data = taxi_entrenar)" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe-in-tidymodels", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#using-a-recipe-in-tidymodels", - "title": "Extras - Recipes", - "section": "Using a recipe in tidymodels", - "text": "Using a recipe in tidymodels\nThe recommended way to use a recipe in tidymodels is to use it as part of a workflow().\n\nforested_wflow <- workflow() |> \n add_recipe(forested_rec) |> \n add_model(linear_reg())\n\nWhen used in this way, you don’t need to worry about prep() and bake() as it is handled for you." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-3", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-3", + "title": "3 - Las partes de un modelo", + "section": "Tu turno", + "text": "Tu turno\n\nCorre:\npredict(arbol_flujo, new_data = taxi_prueba)\n¿Que resultado te dio?\n\n\n\n−+\n03:00" }, { - "objectID": "archive/2025-08-nyr/slides/intro-extra-recipes.html#more-information", - "href": "archive/2025-08-nyr/slides/intro-extra-recipes.html#more-information", - "title": "Extras - Recipes", - "section": "More information", - "text": "More information\n\nhttps://recipes.tidymodels.org/\nhttps://www.tmwr.org/recipes.html" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-4", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-4", + "title": "3 - Las partes de un modelo", + "section": "Tu turno", + "text": "Tu turno\n\nCorre:\naugment(arbol_flujo, new_data = taxi_prueba)\n¿Que resultado te dio?\n\n\n\n−+\n03:00" }, { - "objectID": "slides/annotations.html#understand-your-model", - "href": "slides/annotations.html#understand-your-model", - "title": "Annotations", - "section": "Understand your model", - "text": "Understand your model\nWe’ve abbreviated the code necessary to make the exact plot on the previous slide. The full code is:\nlibrary(rpart.plot)\nsplit_fun <- function(x, labs, digits, varlen, faclen) {\n for (i in 1:length(labs)) {\n if (grepl(\",\", labs[i])) {\n parts <- strsplit(labs[i], \",\")[[1]]\n parts <- trimws(parts)\n if (length(parts) > 5) {\n parts <- c(parts[1:5], \"...\")\n }\n labs[i] <- paste(parts, collapse = \", \")\n }\n labs[i] <- paste(strwrap(labs[i], width = 15), collapse = \"\\n\")\n }\n labs\n}\n\ntree_fit |>\n extract_fit_engine() |>\n rpart.plot(\n roundint = FALSE,\n type = 3,\n clip.right.labs = FALSE,\n split.fun = split_fun\n )" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo", + "title": "3 - Las partes de un modelo", + "section": "Entiendedo tú modelo ", + "text": "Entiendedo tú modelo \n¿Como podemos entender nuestro nuevo modelo arbol_flujo?" }, { - "objectID": "archive/2025-08-nyr/index.html", - "href": "archive/2025-08-nyr/index.html", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels offered at the 2025 New York Data Science & AI Conference. The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to Machine Learning in R with tidymodels and Getting More Out of Feature Engineering and Tuning for Machine Learning courses.\nIntroduction to Machine Learning in R with tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nGetting More Out of Feature Engineering and Tuning for Machine Learning will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo-1", + "title": "3 - Las partes de un modelo", + "section": "Entiendedo tú modelo ", + "text": "Entiendedo tú modelo \n¿Como podemos entender nuestro nuevo modelo arbol_flujo?\n\nlibrary(rpart.plot)\narbol_flujo %>%\n extract_fit_engine() %>%\n rpart.plot(roundint = FALSE)\n\nPuedes extraer varios componentes de tu flujo usando las funciones que empiezan con extract_*()\n\n⚠️ ¡Nunca trate de predecir usando los componentes extraídos!\n\nroundint = FALSE is only to quiet a warning" }, { - "objectID": "archive/2025-08-nyr/index.html#welcome", - "href": "archive/2025-08-nyr/index.html#welcome", - "title": "Machine learning with tidymodels", - "section": "", - "text": "These are the materials for workshops on tidymodels offered at the 2025 New York Data Science & AI Conference. The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to Machine Learning in R with tidymodels and Getting More Out of Feature Engineering and Tuning for Machine Learning courses.\nIntroduction to Machine Learning in R with tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nGetting More Out of Feature Engineering and Tuning for Machine Learning will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo-2", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#entiendedo-tú-modelo-2", + "title": "3 - Las partes de un modelo", + "section": "Entiendedo tú modelo ", + "text": "Entiendedo tú modelo \n¿Como podemos entender nuestro nuevo modelo arbol_flujo?\n\nPuedes usar tu flujo ajustado para obtener explicaciones del modelo o de las predicciones:\n\n\n\nEl paquete vip puede dar la importancia de cada variable\n\n\n\n\nPara obtener explicaciones del modelo, puede usar el paquete DALEXtra\n\n\n\nPara aprender más: https://www.tmwr.org/explain.html" }, { - "objectID": "archive/2025-08-nyr/index.html#is-this-workshop-for-me", - "href": "archive/2025-08-nyr/index.html#is-this-workshop-for-me", - "title": "Machine learning with tidymodels", - "section": "Is this workshop for me? ", - "text": "Is this workshop for me? \nDepending on your background, one of Introduction to Machine Learning in R with tidymodels or Getting More Out of Feature Engineering and Tuning for Machine Learning might serve you better than the other.\n\nIntroduction to Machine Learning in R with tidymodels\nThis workshop is for you if you:\n\nare comfortable using tidyverse packages to read data into R, transform and reshape data, and make a variety of graphs, and\nhave had some exposure to basic statistical concepts such as linear models, residuals, etc.\n\nIntermediate or expert familiarity with modeling or machine learning is not required. Interested students who have intermediate or expert familiarity with modeling or machine learning may be interested in the Getting More Out of Feature Engineering and Tuning for Machine Learning workshop.\n\n\nGetting More Out of Feature Engineering and Tuning for Machine Learning\nThis workshop is for you if you:\n\nhave the prerequisite skills listed for the Introduction to Machine Learning in R with tidymodels workshops,\nhave used tidymodels packages like recipes, rsample, and parsnip, and\nhave some experience with evaluating statistical models using resampling techniques like v-fold cross-validation or the bootstrap.\n\nParticipants who are new to tidymodels or machine learning will benefit from taking the Introduction to Machine Learning in R with tidymodels workshop before joining this one. Participants who have completed the “Introduction to Machine Learning in R with tidymodels” workshop previously will be well-prepared for this course." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-5", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#tu-turno-5", + "title": "3 - Las partes de un modelo", + "section": "Tu turno", + "text": "Tu turno\n\n\nExtrae el “motor” del modelo de tu flujo y examínalo\n\n\n\n−+\n05:00\n\n\n\n\nAfterward, ask what kind of object people got from the extraction, and what they did with it (e.g. give it to summary(), plot(), broom::tidy() ). Live code along" }, { - "objectID": "archive/2025-08-nyr/index.html#preparation", - "href": "archive/2025-08-nyr/index.html#preparation", - "title": "Machine learning with tidymodels", - "section": "Preparation", - "text": "Preparation\nThe process to set up your computer for either workshop will look the same. Please join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2024.04.0), available at https://posit.co/download/rstudio-desktop/\nFor all of the slides, the following R packages can be installed from the R console:\n\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\",\n \"forested\", \"lightgbm\", \"lme4\", \"pak\", \"parallelly\", \"plumber\", \n \"probably\", \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \n \"stacks\", \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nIf you’re a Windows user and encounter an error message during installation noting a missing Rtools installation, install Rtools using the installer linked here." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#expectativas-del-taller---donde-estamos", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-03-partes-del-modelo.html#expectativas-del-taller---donde-estamos", + "title": "3 - Las partes de un modelo", + "section": "Expectativas del taller - Donde estamos", + "text": "Expectativas del taller - Donde estamos\n\n\n\n\n\nflowchart LR\n ad[Todos\\nlos datos]\n style ad fill:#fff,stroke:#666,color:#000\n tr[Entrenamiento]\n style tr fill:#FBE9BF,stroke:#666,color:#000\n ts[Prueba]\n style ts fill:#E5E7FD,stroke:#666,color:#000\n ad --> tr\n ad --> ts\n rs[Remuestreo]\n style rs fill:#fff,stroke:#eee,color:#ddd\n tr --> dt\n lg[Regresión\\nlogística]\n style lg fill:#fff,stroke:#eee,color:#ddd\n rs --> lg\n dt[Arbol de\\nDecisión]\n style dt fill:#FDF4E3,stroke:#666,color:#000\n rs --> dt\n rf[Bosque\\nAleatorio]\n style rf fill:#fff,stroke:#eee,color:#ddd\n rs --> rf\n sm[Seleccionar\\nmodelo]\n style sm fill:#fff,stroke:#eee,color:#ddd\n lg --> sm\n dt --> sm\n rf --> sm\n fm[Entrenar modelo\\nselecionado]\n style fm fill:#fff,stroke:#eee,color:#ddd\n sm --> fm\n tr --> fm\n vm[Verificar la\\ncalidad]\n style vm fill:#fff,stroke:#eee,color:#ddd\n fm --> vm\n ts --> vm\n\n\n\n\n\n\n\n\nStress that fitting a model on the entire training set was only for illustrating how to fit a model" }, { - "objectID": "archive/2025-08-nyr/index.html#slides", - "href": "archive/2025-08-nyr/index.html#slides", - "title": "Machine learning with tidymodels", - "section": "Slides", - "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. They are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to Machine Learning in R with tidymodels\n\n\nGetting More Out of Feature Engineering and Tuning for Machine Learning\n\n\nExtra content (time permitting)" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#parámetros-de-afinamiento", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#parámetros-de-afinamiento", + "title": "5 - Afinando modelos", + "section": "Parámetros de afinamiento", + "text": "Parámetros de afinamiento\nAlgunos valores de los parametros de los modelos o de los preprocesadores no se pueden estimar directamente desde los datos\n\nPor ejemplo:\n\nLa profundidad de los árboles de decisión\nLa cantidad de vecinos en un modelo de vecinos K-nearest" }, { - "objectID": "archive/2025-08-nyr/index.html#code", - "href": "archive/2025-08-nyr/index.html#code", - "title": "Machine learning with tidymodels", - "section": "Code", - "text": "Code\nQuarto files for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#optimiza-los-parámetros-de-afinamiento", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#optimiza-los-parámetros-de-afinamiento", + "title": "5 - Afinando modelos", + "section": "Optimiza los parámetros de afinamiento", + "text": "Optimiza los parámetros de afinamiento\n\nPrueba valores diferentes para medir la calidad del modelo\n\n\n\nEncuentra buenos valores para los parametros\n\n\n\n\nUna vez los valores de los parametros han sido determinados, el modelo final se puede ajustar usando todos los datos en el set de entrenamiento" }, { - "objectID": "archive/2025-08-nyr/index.html#past-workshops", - "href": "archive/2025-08-nyr/index.html#past-workshops", - "title": "Machine learning with tidymodels", - "section": "Past workshops", - "text": "Past workshops\n\nEnglish\n\nAugust 2024 at posit::conf()\nSeptember 2023 at posit::conf()\nJuly 2023 at the New York R Conference\nAugust 2022 in Reykjavik\nJuly 2022 at rstudio::conf()\n\n\n\nSpanish\n\nMarch 2024 at conectaR" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#optimiza-los-parámetros-de-afinamiento-1", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#optimiza-los-parámetros-de-afinamiento-1", + "title": "5 - Afinando modelos", + "section": "Optimiza los parámetros de afinamiento", + "text": "Optimiza los parámetros de afinamiento\nHay dos estrategias principales para optimizar:\n\n\nBusqueda de cuadrícula (Grid search) 💠 que prueba un set de valores pre-elejidos\nBusqueda iterativa 🌀 que sugiere nuevos valores del parametro para probar" }, { - "objectID": "archive/2025-08-nyr/index.html#acknowledgments", - "href": "archive/2025-08-nyr/index.html#acknowledgments", - "title": "Machine learning with tidymodels", - "section": "Acknowledgments", - "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#especificando-los-parametros-de-afinamiento", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#especificando-los-parametros-de-afinamiento", + "title": "5 - Afinando modelos", + "section": "Especificando los parametros de afinamiento", + "text": "Especificando los parametros de afinamiento\nTomemos nueso modelo de bosque aleatorio y tratemos de encontrar el número mínimo de datos min_n\n\narbol_espec <- rand_forest(min_n = tune()) %>% \n set_mode(\"classification\")\n\narbol_flujo <- workflow(propina ~ ., arbol_espec)\narbol_flujo\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> propina ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> min_n = tune()\n#> \n#> Computational engine: ranger" }, { - "objectID": "archive/2025-08-nyr/index.html#reuse-and-licensing", - "href": "archive/2025-08-nyr/index.html#reuse-and-licensing", - "title": "Machine learning with tidymodels", - "section": "Reuse and licensing", - "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#try-out-multiple-values", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#try-out-multiple-values", + "title": "5 - Afinando modelos", + "section": "Try out multiple values", + "text": "Try out multiple values" }, { - "objectID": "slides/intro-extra-recipes.html", - "href": "slides/intro-extra-recipes.html", - "title": "Extras - Recipes", - "section": "", - "text": "#> ── Attaching packages ──────────────────────────── tidymodels 1.3.0 ──\n#> ✔ broom 1.0.9 ✔ rsample 1.3.1 \n#> ✔ dials 1.4.1.9000 ✔ tibble 3.3.0 \n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1 \n#> ✔ infer 1.0.9 ✔ tune 2.0.0 \n#> ✔ modeldata 1.5.1 ✔ workflows 1.3.0.9000\n#> ✔ parsnip 1.3.3 ✔ workflowsets 1.1.1 \n#> ✔ purrr 1.1.0 ✔ yardstick 1.3.2 \n#> ✔ recipes 1.3.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n\n\nforested_train\n#> # A tibble: 8,749 × 19\n#> forested year elevation eastness roughness tree_no_tree dew_temp precip_annual temp_annual_mean temp_annual_min temp_annual_max temp_january_min vapor_min vapor_max canopy_cover lon lat\n#> <fct> <dbl> <dbl> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 1997 66 82 10 Tree 12.2 1315 18.4 1.88 24.9 11.8 121 1888 66 -84.9 32.4\n#> 2 No 1997 284 -99 58 Tree 10.3 1236 16.1 -0.26 22.3 9.92 68 1586 80 -85.0 34.1\n#> 3 Yes 2022 130 86 15 Tree 11.8 1194 17.6 1.3 24.1 11.2 61 1753 96 -83.0 33.1\n#> 4 Yes 2021 202 -55 3 Tree 10.7 1235 16.6 0.05 23.0 10.2 72 1682 65 -83.0 33.9\n#> 5 Yes 1995 75 -89 1 Tree 13.8 1256 19.2 3.63 25.5 12.9 57 1796 88 -83.4 31.3\n#> 6 No 1995 110 -53 5 Tree 12.4 1236 18.6 2.53 24.8 12.4 102 1835 51 -83.9 32.2\n#> 7 Yes 2022 111 73 12 Tree 11.5 1168 17.4 1 24.0 10.9 67 1772 84 -82.2 33.6\n#> 8 Yes 1997 230 96 14 Tree 9.98 1373 15.4 -1.35 21.8 9.03 46 1552 68 -85.3 34.8\n#> 9 Yes 2002 160 -88 13 Tree 11.1 1219 16.9 0.07 23.6 10.2 53 1731 95 -83.2 33.6\n#> 10 Yes 2020 39 9 6 Tree 13.9 1237 19.2 3.25 25.6 12.8 58 1812 86 -82.0 31.7\n#> # ℹ 8,739 more rows\n#> # ℹ 2 more variables: land_type <fct>, county <fct>" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#prueba-varios-valores", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#prueba-varios-valores", + "title": "5 - Afinando modelos", + "section": "Prueba varios valores", + "text": "Prueba varios valores\ntune_grid() funciona parecido a fit_resamples() pero puede calcular multiples valores del parámetro:\n\nset.seed(22)\nrf_res <- tune_grid(\n arbol_flujo,\n taxi_plieges,\n grid = 5\n)" }, { - "objectID": "slides/intro-05-tuning-models.html", - "href": "slides/intro-05-tuning-models.html", - "title": "5 - Tuning models", - "section": "", - "text": "#> ── Attaching packages ──────────────────────────── tidymodels 1.3.0 ──\n#> ✔ broom 1.0.9 ✔ rsample 1.3.1 \n#> ✔ dials 1.4.1.9000 ✔ tibble 3.3.0 \n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1 \n#> ✔ infer 1.0.9 ✔ tune 2.0.0 \n#> ✔ modeldata 1.5.1 ✔ workflows 1.3.0.9000\n#> ✔ parsnip 1.3.3 ✔ workflowsets 1.1.1 \n#> ✔ purrr 1.1.0 ✔ yardstick 1.3.2 \n#> ✔ recipes 1.3.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#compara-los-resultados", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#compara-los-resultados", + "title": "5 - Afinando modelos", + "section": "Compara los resultados", + "text": "Compara los resultados\nInspecciona los resultados, y selecciona los mejores parámetros:\n\nshow_best(rf_res)\n#> # A tibble: 5 × 7\n#> min_n .metric .estimator mean n std_err .config \n#> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 33 roc_auc binary 0.653 10 0.0159 Preprocessor1_Model1\n#> 2 31 roc_auc binary 0.650 10 0.0167 Preprocessor1_Model3\n#> 3 21 roc_auc binary 0.646 10 0.0165 Preprocessor1_Model4\n#> 4 13 roc_auc binary 0.646 10 0.0143 Preprocessor1_Model5\n#> 5 6 roc_auc binary 0.631 10 0.0140 Preprocessor1_Model2\n\nparametro_mejor <- select_best(rf_res)\nparametro_mejor\n#> # A tibble: 1 × 2\n#> min_n .config \n#> <int> <chr> \n#> 1 33 Preprocessor1_Model1\n\nTambién puedes usar collect_metrics() y autoplot()" }, { - "objectID": "slides/intro-02-data-budget.html#the-training-set-1", - "href": "slides/intro-02-data-budget.html#the-training-set-1", - "title": "2 - Your data budget", - "section": "The training set", - "text": "The training set\n\nforested_train |>\n select(where(is.factor))\n#> # A tibble: 8,202 × 4\n#> forested tree_no_tree land_type county \n#> <fct> <fct> <fct> <fct> \n#> 1 Yes Tree Tree Muscogee \n#> 2 No Tree Tree Polk \n#> 3 Yes Tree Tree Hancock \n#> 4 Yes Tree Tree Oglethorpe\n#> 5 Yes Tree Tree Berrien \n#> 6 No Tree Non-tree vegetation Dooly \n#> 7 Yes Tree Tree Columbia \n#> 8 Yes Tree Tree Walker \n#> 9 Yes Tree Tree Greene \n#> 10 Yes Tree Tree Wayne \n#> # ℹ 8,192 more rows\n\n\n\nThe tree_no_tree and land_type variables are both outputs from other machine learning models. Companies make their own classifications of forest vs. non-forest based on satellite images and then sell them. Those classifications are available at prediction time; the FIA will happily spend the few thousand dollars to buy the data for the whole country." + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#el-ajuste-final", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#el-ajuste-final", + "title": "5 - Afinando modelos", + "section": "El ajuste final", + "text": "El ajuste final\n\narbol_flujo <- finalize_workflow(arbol_flujo, parametro_mejor)\n\najuste_final <- last_fit(arbol_flujo, taxi_separar) \n\ncollect_metrics(ajuste_final)\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.912 Preprocessor1_Model1\n#> 2 roc_auc binary 0.636 Preprocessor1_Model1" }, { - "objectID": "slides/advanced-01-introduction.html", - "href": "slides/advanced-01-introduction.html", - "title": "1 - Introduction", - "section": "", - "text": "Welcome!" + "objectID": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#tu-turno", + "href": "archive/2024-03-conectaR-spanish/presentacion/intro-05-afinando-modelos.html#tu-turno", + "title": "5 - Afinando modelos", + "section": "Tu turno", + "text": "Tu turno\n\nModifica tu flujo de modelo para usar por lo menos un parámetro\nUsa busqueda de cuadricula para encontrar los mejores parametros\n\n\n\n−+\n05:00" }, { - "objectID": "slides/advanced-01-introduction.html#venue-information", - "href": "slides/advanced-01-introduction.html#venue-information", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#venue-information", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#venue-information", "title": "1 - Introduction", "section": "Venue information", - "text": "Venue information\n\nThere are gender neutral bathrooms located on floor LL2, next to Chicago A\nA meditation/prayer room is located on floor LL2 in Chicago A\nA lactation room is located on floor LL2 in Chicago B" + "text": "Venue information\n\nThere are gender neutral bathrooms located on levels 3, 4, 5, 6 & 7\nA meditation/prayer room is located in 503\n(Mon & Tue 7am - 7pm, and Wed 7am - 5pm)\nA lactation room is located in 509\n(Mon & Tue 7am - 7pm, and Wed 7am - 5pm)" }, { - "objectID": "slides/advanced-01-introduction.html#workshop-policies", - "href": "slides/advanced-01-introduction.html#workshop-policies", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#workshop-policies", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#workshop-policies", "title": "1 - Introduction", "section": "Workshop policies", "text": "Workshop policies\n\nPlease review the posit::conf code of conduct, which applies to all workshops: https://posit.co/code-of-conduct\nCoC site has info on how to report a problem (in person, email, phone)\nPlease do not photograph people wearing red lanyards" }, { - "objectID": "slides/advanced-01-introduction.html#asking-for-help", - "href": "slides/advanced-01-introduction.html#asking-for-help", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#who-are-you", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#who-are-you", + "title": "1 - Introduction", + "section": "Who are you?", + "text": "Who are you?\n\nYou can use the magrittr %>% or base R |> pipe\nYou are familiar with functions from dplyr, tidyr, ggplot2\nYou have exposure to basic statistical concepts\nYou do not need intermediate or expert familiarity with modeling or ML\nYou have used some tidymodels packages\nYou have some experience with evaluating statistical models using resampling techniques" + }, + { + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#who-are-tidymodels", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#who-are-tidymodels", + "title": "1 - Introduction", + "section": "Who are tidymodels?", + "text": "Who are tidymodels?\n\nSimon Couch\nHannah Frick\nEmil Hvitfeldt\nMax Kuhn\n\n\nMany thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!" + }, + { + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#asking-for-help", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#asking-for-help", "title": "1 - Introduction", "section": "Asking for help", "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" }, { - "objectID": "slides/advanced-01-introduction.html#discord", - "href": "slides/advanced-01-introduction.html#discord", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#discord", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#discord", "title": "1 - Introduction", "section": "Discord ", - "text": "Discord \n\npos.it/conf-event-portal (login)\nClick on “Join Discord, the virtual networking platform!”\nBrowse Channels -> #workshop-feat-eng-tune" - }, - { - "objectID": "slides/advanced-06-wrapping-up.html", - "href": "slides/advanced-06-wrapping-up.html", - "title": "6 - Wrapping up", - "section": "", - "text": "We made it!" + "text": "Discord \n\npos.it/conf-event-portal (login)\nClick on “Join Discord, the virtual networking platform!”\nBrowse Channels -> #workshop-tidymodels-advanced" }, { - "objectID": "slides/intro-04-evaluating-models.html", - "href": "slides/intro-04-evaluating-models.html", - "title": "4 - Evaluating models", - "section": "", - "text": "#> ── Attaching packages ──────────────────────────── tidymodels 1.3.0 ──\n#> ✔ broom 1.0.9 ✔ rsample 1.3.1 \n#> ✔ dials 1.4.1.9000 ✔ tibble 3.3.0 \n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1 \n#> ✔ infer 1.0.9 ✔ tune 2.0.0 \n#> ✔ modeldata 1.5.1 ✔ workflows 1.3.0.9000\n#> ✔ parsnip 1.3.3 ✔ workflowsets 1.1.1 \n#> ✔ purrr 1.1.0 ✔ yardstick 1.3.2 \n#> ✔ recipes 1.3.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()\n\n\naugment(forested_fit, new_data = forested_train)\n#> # A tibble: 8,749 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness roughness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 Yes 0.931 0.0690 Yes 1997 66 82 10\n#> 2 Yes 0.983 0.0172 No 1997 284 -99 58\n#> 3 Yes 0.960 0.0401 Yes 2022 130 86 15\n#> 4 Yes 0.870 0.130 Yes 2021 202 -55 3\n#> 5 Yes 0.823 0.177 Yes 1995 75 -89 1\n#> 6 Yes 0.758 0.242 No 1995 110 -53 5\n#> 7 Yes 0.823 0.177 Yes 2022 111 73 12\n#> 8 No 0.467 0.533 Yes 1997 230 96 14\n#> 9 Yes 0.983 0.0172 Yes 2002 160 -88 13\n#> 10 Yes 0.871 0.129 Yes 2020 39 9 6\n#> # ℹ 8,739 more rows\n#> # ℹ 14 more variables: tree_no_tree <fct>, dew_temp <dbl>, precip_annual <dbl>,\n#> # temp_annual_mean <dbl>, temp_annual_min <dbl>, temp_annual_max <dbl>,\n#> # temp_january_min <dbl>, vapor_min <dbl>, vapor_max <dbl>,\n#> # canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>, county <fct>" + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-2", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-2", + "title": "1 - Introduction", + "section": "👀", + "text": "👀" }, { - "objectID": "slides/intro-04-evaluating-models.html#dangers-of-accuracy", - "href": "slides/intro-04-evaluating-models.html#dangers-of-accuracy", - "title": "4 - Evaluating models", - "section": "Dangers of accuracy ", - "text": "Dangers of accuracy \nWe need to be careful of using accuracy() since it can give “good” performance by only predicting one way with imbalanced data:\n\naugment(forested_fit, new_data = forested_train) %>%\n mutate(.pred_class = factor(\"Yes\", levels = c(\"Yes\", \"No\"))) %>%\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.723" + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-3", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-3", + "title": "1 - Introduction", + "section": "👀", + "text": "👀\n\n\nhttps://workshops.tidymodels.org" }, { - "objectID": "slides/intro-04-evaluating-models.html#your-turn-6", - "href": "slides/intro-04-evaluating-models.html#your-turn-6", - "title": "4 - Evaluating models", - "section": "Your turn", - "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#tentative-plan-for-this-workshop", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#tentative-plan-for-this-workshop", + "title": "1 - Introduction", + "section": "Tentative plan for this workshop", + "text": "Tentative plan for this workshop\n\nFeature engineering with recipes\nModel optimization by tuning\n\nGrid search\nRacing\nIterative methods\n\nExtras (time permitting)\n\nEffect encodings\nA case study" }, { - "objectID": "slides/intro-06-wrapping-up.html", - "href": "slides/intro-06-wrapping-up.html", - "title": "6 - Wrapping up", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-4", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#section-4", + "title": "1 - Introduction", "section": "", - "text": "We made it!" + "text": "Introduce yourself to your neighbors 👋\n\n Log in to Posit Cloud (free): TODO-ADD-LATER" }, { - "objectID": "slides/intro-extra-workflowsets.html", - "href": "slides/intro-extra-workflowsets.html", - "title": "Extras - workflowsets", - "section": "", - "text": "#> ── Attaching packages ──────────────────────────── tidymodels 1.3.0 ──\n#> ✔ broom 1.0.9 ✔ rsample 1.3.1 \n#> ✔ dials 1.4.1.9000 ✔ tibble 3.3.0 \n#> ✔ dplyr 1.1.4 ✔ tidyr 1.3.1 \n#> ✔ infer 1.0.9 ✔ tune 2.0.0 \n#> ✔ modeldata 1.5.1 ✔ workflows 1.3.0.9000\n#> ✔ parsnip 1.3.3 ✔ workflowsets 1.1.1 \n#> ✔ purrr 1.1.0 ✔ yardstick 1.3.2 \n#> ✔ recipes 1.3.1\n#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──\n#> ✖ purrr::discard() masks scales::discard()\n#> ✖ dplyr::filter() masks stats::filter()\n#> ✖ dplyr::lag() masks stats::lag()\n#> ✖ recipes::step() masks stats::step()" + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#lets-install-some-packages", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#lets-install-some-packages", + "title": "1 - Introduction", + "section": "Let’s install some packages", + "text": "Let’s install some packages\nIf you are using your own laptop instead of Posit Cloud:\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"forested\", \"lightgbm\", \"lme4\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nAlso, you should install the newest version of the dials package (version 1.3.0). To check this, you can run:\n\nrlang::check_installed(\"dials\", version = \"1.3.0\")" }, { - "objectID": "slides/intro-01-introduction.html#venue-information", - "href": "slides/intro-01-introduction.html#venue-information", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#hotel-data", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#hotel-data", "title": "1 - Introduction", - "section": "Venue information", - "text": "Venue information\n\nThere are gender neutral bathrooms located on floor LL2, next to Chicago A\nA meditation/prayer room is located on floor LL2 in Chicago A\nA lactation room is located on floor LL2 in Chicago B" + "section": "Hotel Data ", + "text": "Hotel Data \nWe’ll use data on hotels to predict the cost of a room.\nThe data are in the modeldata package. We’ll sample down the data and refactor some columns:\n\n\n\nlibrary(tidymodels)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "slides/intro-01-introduction.html#workshop-policies", - "href": "slides/intro-01-introduction.html#workshop-policies", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#hotel-date-columns", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#hotel-date-columns", "title": "1 - Introduction", - "section": "Workshop policies", - "text": "Workshop policies\n\nPlease review the posit::conf code of conduct, which applies to all workshops: https://posit.co/code-of-conduct\nCoC site has info on how to report a problem (in person, email, phone)\nPlease do not photograph people wearing red lanyards" + "section": "Hotel date columns", + "text": "Hotel date columns\n\nnames(hotel_rates)\n#> [1] \"avg_price_per_room\" \"lead_time\" \n#> [3] \"stays_in_weekend_nights\" \"stays_in_week_nights\" \n#> [5] \"adults\" \"children\" \n#> [7] \"babies\" \"meal\" \n#> [9] \"country\" \"market_segment\" \n#> [11] \"distribution_channel\" \"is_repeated_guest\" \n#> [13] \"previous_cancellations\" \"previous_bookings_not_canceled\"\n#> [15] \"reserved_room_type\" \"assigned_room_type\" \n#> [17] \"booking_changes\" \"agent\" \n#> [19] \"company\" \"days_in_waiting_list\" \n#> [21] \"customer_type\" \"required_car_parking_spaces\" \n#> [23] \"total_of_special_requests\" \"arrival_date_num\" \n#> [25] \"near_christmas\" \"near_new_years\" \n#> [27] \"historical_adr\"" }, { - "objectID": "slides/intro-01-introduction.html#asking-for-help", - "href": "slides/intro-01-introduction.html#asking-for-help", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#data-splitting-strategy", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#data-splitting-strategy", "title": "1 - Introduction", - "section": "Asking for help", - "text": "Asking for help\n\n🟪 “I’m stuck and need help!”\n\n\n🟩 “I finished the exercise”" + "section": "Data splitting strategy", + "text": "Data splitting strategy" }, { - "objectID": "slides/intro-01-introduction.html#discord", - "href": "slides/intro-01-introduction.html#discord", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#data-spending", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#data-spending", "title": "1 - Introduction", - "section": "Discord ", - "text": "Discord \n\npos.it/conf-event-portal (login)\nClick on “Join Discord, the virtual networking platform!”\nBrowse Channels -> #workshop-tidymodels" + "section": "Data Spending ", + "text": "Data Spending \nLet’s split the data into a training set (75%) and testing set (25%) using stratification:\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" }, { - "objectID": "slides/intro-01-introduction.html#what-is-machine-learning-2025-edition", - "href": "slides/intro-01-introduction.html#what-is-machine-learning-2025-edition", + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#your-turn", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#your-turn", "title": "1 - Introduction", - "section": "What is machine learning? (2025 edition)", - "text": "What is machine learning? (2025 edition)\n\n\nIllustration credit: https://vas3k.com/blog/machine_learning/\n\n\nIn the early 2010s, “Artificial intelligence” (AI) was largely synonymous with what we’ll refer to as “machine learning” in this workshop. In the late 2010s and early 2020s, AI usually referred to deep learning methods. Since the release of ChatGPT in late 2022, “AI” has come to also encompass large language models / generative models." + "section": "Your turn", + "text": "Your turn\nLet’s take some time and investigate the training data. The outcome is avg_price_per_room.\nAre there any interesting characteristics of the data?\n\n\n\n−+\n10:00" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#getting-set-up", - "href": "slides/advanced-04-feature-engineering-part-two.html#getting-set-up", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Getting set up ", - "text": "Getting set up \n\nlibrary(tidymodels)\nlibrary(embed)\nlibrary(extrasteps)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)" + "objectID": "archive/2024-08-posit-conf/advanced-01-introduction.html#our-versions", + "href": "archive/2024-08-posit-conf/advanced-01-introduction.html#our-versions", + "title": "1 - Introduction", + "section": "Our versions", + "text": "Our versions\nR version 4.4.1 (2024-06-14), Quarto (1.6.1)\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nbonsai\n0.3.1\n\n\nbroom\n1.0.6\n\n\nCubist\n0.4.4\n\n\ndials\n1.3.0\n\n\ndoParallel\n1.0.17\n\n\ndplyr\n1.1.4\n\n\nearth\n5.3.3\n\n\nembed\n1.1.4\n\n\nfinetune\n1.2.0\n\n\nFormula\n1.2-5\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nggplot2\n3.5.1\n\n\nlattice\n0.22-6\n\n\nlightgbm\n4.5.0\n\n\nlme4\n1.1-35.5\n\n\nmodeldata\n1.4.0\n\n\nparallelly\n1.38.0\n\n\nparsnip\n1.2.1\n\n\nplotmo\n3.6.3\n\n\nplotrix\n3.8-4\n\n\nplumber\n1.2.2\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\nprobably\n1.0.3\n\n\npurrr\n1.0.2\n\n\nrecipes\n1.1.0\n\n\nrsample\n1.2.1\n\n\nrules\n1.0.2\n\n\nscales\n1.3.0\n\n\nsplines2\n0.5.3\n\n\nstacks\n1.0.5\n\n\ntext2vec\n0.6.4\n\n\ntextrecipes\n1.0.6\n\n\n\n\n\n\n\n\n\n\n\npackage\nversion\n\n\n\n\ntibble\n3.2.1\n\n\ntidymodels\n1.2.0\n\n\ntidyr\n1.3.1\n\n\ntune\n1.2.1\n\n\nvetiver\n0.2.5\n\n\nworkflows\n1.1.4\n\n\nworkflowsets\n1.1.0\n\n\nyardstick\n1.3.1" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#hotel-rates-data-set", - "href": "slides/advanced-04-feature-engineering-part-two.html#hotel-rates-data-set", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Hotel rates data set", - "text": "Hotel rates data set\nRegression data set for predicting the average daily rate for a room, for “Resort Hotel”. The agent and company use random names.\n\nglimpse(hotel_rates)\n#> Rows: 15,402\n#> Columns: 28\n#> $ avg_price_per_room <dbl> 110.00, 74.00, 81.90, 81.00, 112.20, 90…\n#> $ lead_time <dbl> 241, 273, 248, 236, 243, 267, 94, 10, 1…\n#> $ stays_in_weekend_nights <dbl> 0, 2, 2, 2, 4, 2, 4, 0, 0, 0, 0, 0, 0, …\n#> $ stays_in_week_nights <dbl> 1, 5, 5, 5, 10, 5, 7, 1, 1, 1, 1, 1, 1,…\n#> $ adults <dbl> 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, …\n#> $ children <dbl> 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, …\n#> $ babies <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ meal <fct> bed_and_breakfast, bed_and_breakfast, b…\n#> $ country <fct> prt, aus, gbr, prt, gbr, null, prt, esp…\n#> $ market_segment <fct> online_travel_agent, offline_travel_age…\n#> $ distribution_channel <fct> ta_to, ta_to, ta_to, ta_to, ta_to, ta_t…\n#> $ is_repeated_guest <dbl> 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, …\n#> $ previous_cancellations <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ previous_bookings_not_canceled <dbl> 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, …\n#> $ reserved_room_type <fct> a, a, a, a, a, a, f, e, h, a, a, g, a, …\n#> $ assigned_room_type <fct> c, a, c, a, a, a, f, f, h, e, e, g, e, …\n#> $ booking_changes <dbl> 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, …\n#> $ agent <fct> devin_rivera_borrego, lia_nauth, jawhar…\n#> $ company <fct> not_applicable, not_applicable, not_app…\n#> $ days_in_waiting_list <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ customer_type <fct> transient, transient_party, transient, …\n#> $ required_car_parking_spaces <dbl> 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, …\n#> $ total_of_special_requests <dbl> 1, 0, 0, 2, 0, 0, 1, 1, 0, 2, 2, 0, 2, …\n#> $ arrival_date <date> 2016-07-02, 2016-07-02, 2016-07-02, 20…\n#> $ arrival_date_num <dbl> 2016.5, 2016.5, 2016.5, 2016.5, 2016.5,…\n#> $ near_christmas <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ near_new_years <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ historical_adr <dbl> 104.9811, 104.9811, 104.9811, 104.9811,…\n\n\nAntonio, N., de Almeida, A., and Nunes, L. (2019). Hotel booking demand datasets. Data in Brief, 22, 41-49." + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---setup", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---setup", + "title": "3 - Tuning Hyperparameters", + "section": "Previously - Setup ", + "text": "Previously - Setup \n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\nreg_metrics <- metric_set(mae, rsq)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#hotel-data-splitting", - "href": "slides/advanced-04-feature-engineering-part-two.html#hotel-data-splitting", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Hotel data splitting ", - "text": "Hotel data splitting \nGenerally, you should always do data splitting. We are doing it here explicitly because some artifacts of splitting data become useful later on.\n\nset.seed(1234)\nhotel_split <- initial_split(hotel_rates)\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---data-usage", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---data-usage", + "title": "3 - Tuning Hyperparameters", + "section": "Previously - Data Usage ", + "text": "Previously - Data Usage \n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#your-turn", - "href": "slides/advanced-04-feature-engineering-part-two.html#your-turn", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Your turn", - "text": "Your turn\n\nLoad and explore the hotel_train data\nComes loaded with the modeldata package\n\n\n\n−+\n05:00" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#previously---feature-engineering", + "title": "3 - Tuning Hyperparameters", + "section": "Previously - Feature engineering ", + "text": "Previously - Feature engineering \n\nlibrary(textrecipes)\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n # Defaults to 32 signed indicator columns\n step_dummy_hash(agent) %>%\n step_dummy_hash(company) %>%\n # Regular indicators for the others\n step_dummy(all_nominal_predictors()) %>% \n step_zv(all_predictors())" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#nonlinear-predictors", - "href": "slides/advanced-04-feature-engineering-part-two.html#nonlinear-predictors", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Nonlinear predictors", - "text": "Nonlinear predictors\n\n\nFigure 1" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-parameters", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Tuning parameters", + "text": "Tuning parameters\nSome model or preprocessing parameters cannot be estimated directly from the data.\n\nSome examples:\n\nTree depth in decision trees\nNumber of neighbors in a K-nearest neighbor model" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#splines-1", - "href": "slides/advanced-04-feature-engineering-part-two.html#splines-1", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Splines", - "text": "Splines\nIt is a way to transform a single numeric predictor into multiple numeric predictors, with the hope that the new numeric predictors are more linearly related to the outcome.\nMostly needed with linear models, but it should rarely hurt to use it.\n\nFEAZ FES" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\n\nTry different values and measure their performance.\n\n\n\nFind good values for these parameters.\n\n\n\n\nOnce the value(s) of the parameter(s) are determined, a model can be finalized by fitting the model to the entire training set." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#splines-explained", - "href": "slides/advanced-04-feature-engineering-part-two.html#splines-explained", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Splines explained", - "text": "Splines explained\nA B-spline is a piecewise polynomial function.\nWe have 2 main parameters to worry about. Number of knots and the polynomial degree.\nThe domain of the predictor is split into k regions, with a knot between each, and a polynomial function is fit within each region, under the constraint that they touch each other at the knot." + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tagging-parameters-for-tuning", + "title": "3 - Tuning Hyperparameters", + "section": "Tagging parameters for tuning ", + "text": "Tagging parameters for tuning \nWith tidymodels, you can mark the parameters that you want to optimize with a value of tune().\n\nThe function itself just returns… itself:\n\ntune()\n#> tune()\nstr(tune())\n#> language tune()\n\n# optionally add a label\ntune(\"I hope that the workshop is going well\")\n#> tune(\"I hope that the workshop is going well\")\n\n\nFor example…" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#knots-1-degree-1", - "href": "slides/advanced-04-feature-engineering-part-two.html#knots-1-degree-1", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "knots: 1, degree: 1", - "text": "knots: 1, degree: 1\n\n\nFigure 2" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimizing-the-hash-features", + "title": "3 - Tuning Hyperparameters", + "section": "Optimizing the hash features ", + "text": "Optimizing the hash features \nOur new recipe is:\n\nhash_rec <-\n recipe(avg_price_per_room ~ ., data = hotel_train) %>%\n step_YeoJohnson(lead_time) %>%\n step_dummy_hash(agent, num_terms = tune(\"agent hash\")) %>%\n step_dummy_hash(company, num_terms = tune(\"company hash\")) %>%\n step_zv(all_predictors())\n\n\nWe will be using a tree-based model in a minute.\n\nThe other categorical predictors are left as-is.\nThat’s why there is no step_dummy()." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#knots-2-degree-1", - "href": "slides/advanced-04-feature-engineering-part-two.html#knots-2-degree-1", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "knots: 2, degree: 1", - "text": "knots: 2, degree: 1\n\n\nFigure 3" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-trees", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-trees", + "title": "3 - Tuning Hyperparameters", + "section": "Boosted Trees", + "text": "Boosted Trees\nThese are popular ensemble methods that build a sequence of tree models.\n\nEach tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.\n\nEach tree in the ensemble is saved and new samples are predicted using a weighted average of the votes of each tree in the ensemble.\n\nWe’ll focus on the popular lightgbm implementation." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#knots-5-degree-1", - "href": "slides/advanced-04-feature-engineering-part-two.html#knots-5-degree-1", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "knots: 5, degree: 1", - "text": "knots: 5, degree: 1\n\n\nFigure 4" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nSome possible parameters:\n\nmtry: The number of predictors randomly sampled at each split (in \\([1, ncol(x)]\\) or \\((0, 1]\\)).\ntrees: The number of trees (\\([1, \\infty]\\), but usually up to thousands)\nmin_n: The number of samples needed to further split (\\([1, n]\\)).\nlearn_rate: The rate that each tree adapts from previous iterations (\\((0, \\infty]\\), usual maximum is 0.1).\nstop_iter: The number of iterations of boosting where no improvement was shown before stopping (\\([1, trees]\\))" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#knots-5-degree-2", - "href": "slides/advanced-04-feature-engineering-part-two.html#knots-5-degree-2", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "knots: 5, degree: 2", - "text": "knots: 5, degree: 2\n\n\nFigure 5" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-1", + "title": "3 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters", + "text": "Boosted Tree Tuning Parameters\nTBH it is usually not difficult to optimize these models.\n\nOften, there are multiple candidate tuning parameter combinations that have very good results.\n\nTo demonstrate simple concepts, we’ll look at optimizing the number of trees in the ensemble (between 1 and 100) and the learning rate (\\(10^{-5}\\) to \\(10^{-1}\\))." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#knots-5-degree-3", - "href": "slides/advanced-04-feature-engineering-part-two.html#knots-5-degree-3", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "knots: 5, degree: 3", - "text": "knots: 5, degree: 3\n\n\nFigure 6" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#boosted-tree-tuning-parameters-2", + "title": "3 - Tuning Hyperparameters", + "section": "Boosted Tree Tuning Parameters ", + "text": "Boosted Tree Tuning Parameters \nWe’ll need to load the bonsai package. This has the information needed to use lightgbm\n\nlibrary(bonsai)\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#knots-9-degree-3", - "href": "slides/advanced-04-feature-engineering-part-two.html#knots-9-degree-3", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "knots: 9, degree: 3", - "text": "knots: 9, degree: 3\n\n\nFigure 7" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#optimize-tuning-parameters-1", + "title": "3 - Tuning Hyperparameters", + "section": "Optimize tuning parameters", + "text": "Optimize tuning parameters\nThe main two strategies for optimization are:\n\n\nGrid search 💠 which tests a pre-defined set of candidate values\nIterative search 🌀 which suggests/estimates new values of candidate parameters to evaluate" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#b-spline-features-visualized---degree-3", - "href": "slides/advanced-04-feature-engineering-part-two.html#b-spline-features-visualized---degree-3", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "B-Spline features visualized - degree: 3", - "text": "B-Spline features visualized - degree: 3" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search", + "title": "3 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nA small grid of points trying to minimize the error via learning rate:" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#splines-as-numbers", - "href": "slides/advanced-04-feature-engineering-part-two.html#splines-as-numbers", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Splines as numbers", - "text": "Splines as numbers\n\n\n\n\n\n\n\n\n\n\n\n\n\n\narrival_date_num\nSpline Feature 1\nSpline Feature 2\nSpline Feature 3\nSpline Feature 4\nSpline Feature 5\nSpline Feature 6\n\n\n\n\n2017.619\n0.00\n0.00\n0.00\n0.03\n0.35\n0.62\n\n\n2016.844\n0.15\n0.59\n0.26\n0.00\n0.00\n0.00\n\n\n2016.702\n0.51\n0.40\n0.05\n0.00\n0.00\n0.00\n\n\n2017.077\n0.00\n0.19\n0.67\n0.14\n0.00\n0.00\n\n\n2016.861\n0.13\n0.58\n0.29\n0.00\n0.00\n0.00\n\n\n2017.019\n0.00\n0.30\n0.62\n0.07\n0.00\n0.00\n\n\n2017.123\n0.00\n0.11\n0.66\n0.23\n0.00\n0.00" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-1", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-1", + "title": "3 - Tuning Hyperparameters", + "section": "Grid search", + "text": "Grid search\nIn reality we would probably sample the space more densely:" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#splines-pros-and-cons", - "href": "slides/advanced-04-feature-engineering-part-two.html#splines-pros-and-cons", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Splines pros and cons", - "text": "Splines pros and cons\n\n\nPros\n\nfast\neasy to use\nsemi-interpretable\n\n\nCons\n\nadds more columns\nwill need to select the number of columns\ncan be messy outside the range" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#iterative-search", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#iterative-search", + "title": "3 - Tuning Hyperparameters", + "section": "Iterative Search", + "text": "Iterative Search\nWe could start with a few points and search the space:" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#your-turn-1", - "href": "slides/advanced-04-feature-engineering-part-two.html#your-turn-1", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Your turn", - "text": "Your turn\n\nApply B-splines to some variables using step_spline_b()\n\n\n\n−+\n03:00" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#parameters", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Parameters", + "text": "Parameters\n\nThe tidymodels framework provides pre-defined information on tuning parameters (such as their type, range, transformations, etc).\nThe extract_parameter_set_dials() function extracts these tuning parameters and the info.\n\n\nGrids\n\nCreate your grid manually or automatically.\nThe grid_*() functions can make a grid.\n\n\n\nMost basic (but very effective) way to tune models" + }, + { + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#different-types-of-grids", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#different-types-of-grids", + "title": "3 - Tuning Hyperparameters", + "section": "Different types of grids ", + "text": "Different types of grids \n\n\n\n\n\n\n\n\n\nSpace-filling designs (SFD) attempt to cover the parameter space without redundant candidates. We recommend these the most." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#factors-with-many-categories", - "href": "slides/advanced-04-feature-engineering-part-two.html#factors-with-many-categories", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Factors with many categories", - "text": "Factors with many categories\n\n\n\nhotel_train |>\n count(country)\n#> # A tibble: 88 × 2\n#> country n\n#> <fct> <int>\n#> 1 ago 7\n#> 2 and 1\n#> 3 are 3\n#> 4 arg 14\n#> 5 aus 31\n#> 6 aut 76\n#> 7 aze 2\n#> 8 bel 184\n#> 9 bgr 2\n#> 10 bhs 1\n#> # ℹ 78 more rows\n\n\n\nhotel_train |>\n count(company)\n#> # A tibble: 157 × 2\n#> company n\n#> <fct> <int>\n#> 1 abdou_llc 2\n#> 2 afework_llc 5\n#> 3 alston_pbc 3\n#> 4 battle_llc 23\n#> 5 bennett_and_company 3\n#> 6 berhanu_pbc 53\n#> 7 biggers_llc 4\n#> 8 blasingime_llc 1\n#> 9 boddy_llc 16\n#> 10 boles_pbc 3\n#> # ℹ 147 more rows\n\n\n\nhotel_train |>\n count(agent)\n#> # A tibble: 119 × 2\n#> agent n\n#> <fct> <int>\n#> 1 aaron_marquez 2\n#> 2 alexander_drake 1117\n#> 3 allen_her 1\n#> 4 anas_el_bashir 1\n#> 5 araseli_billy 1\n#> 6 arhab_al_islam 7\n#> 7 audray_tucker 38\n#> 8 bernice_baltierra 35\n#> 9 betzy_rodriguez 66\n#> 10 brayan_guerrero 2\n#> # ℹ 109 more rows" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid", + "title": "3 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\nlgbm_wflow %>% \n extract_parameter_set_dials()\n#> Collection of 4 parameters for tuning\n#> \n#> identifier type object\n#> trees trees nparam[+]\n#> learn_rate learn_rate nparam[+]\n#> agent hash num_terms nparam[+]\n#> company hash num_terms nparam[+]\n\n# Individual functions: \ntrees()\n#> # Trees (quantitative)\n#> Range: [1, 2000]\nlearn_rate()\n#> Learning Rate (quantitative)\n#> Transformer: log-10 [1e-100, Inf]\n#> Range (transformed scale): [-10, -1]\n\n\nA parameter set can be updated (e.g. to change the ranges)." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#how-do-we-handle-them", - "href": "slides/advanced-04-feature-engineering-part-two.html#how-do-we-handle-them", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "How do we handle them?", - "text": "How do we handle them?\n\nWe could:\n\nMake the full set of indicator variables 😳\nLump agents and companies that rarely occur into an “other” group\nUse feature hashing to create a smaller set of indicator variables\nUse target encoding to replace the county, agent, and company columns with the estimated effect of that predictor" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid-1", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-grid-1", + "title": "3 - Tuning Hyperparameters", + "section": "Create a grid ", + "text": "Create a grid \n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 7.50e- 6 574 574\n#> 2 84 1.78e- 5 2048 2298\n#> 3 167 5.62e-10 1824 912\n#> 4 250 4.22e- 5 3250 512\n#> 5 334 1.78e- 8 512 2896\n#> 6 417 1.33e- 3 322 1625\n#> 7 500 1 e- 1 1448 1149\n#> 8 584 1 e- 7 1290 256\n#> 9 667 2.37e-10 456 724\n#> 10 750 1.78e- 2 645 322\n#> # ℹ 15 more rows" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-1", - "href": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-1", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding", - "text": "Target encoding\nTarget encoding (also called mean encoding, likelihood encoding, impact encoding, or effect encoding) is a supervised trained method that turns a single categorical predictor into a single numeric predictor.\nIt is often used to deal with categorical predictors with many levels, although it works regardless.\nSince it uses the outcome to train it, you need to make sure to use cross-validation to avoid overfitting.\n\nFEAZ FES" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn", + "title": "3 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\nCreate a grid for our tunable workflow.\nTry creating a regular grid.\n\n\n\n−+\n03:00" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-motivation", - "href": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-motivation", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding motivation", - "text": "Target encoding motivation\nYou have a numeric outcome and a categorical predictor. And you want to transform each value of the categorical predictor into a value that best represents the outcome?\n\n\n\nWe calculate the mean of the outcome within each level of the predictor, and use that as the new value.\n\n\n\n\n\n\nCaution\n\n\nDon’t do just this! We are building up the method one thing at a time. Unregularized target encoding is really prone to overfitting.\n\n\n\n\n\nhotel_train |>\n summarise(\n mean = mean(avg_price_per_room),\n .by = agent\n )\n#> # A tibble: 119 × 2\n#> agent mean\n#> <fct> <dbl>\n#> 1 alexander_drake 144. \n#> 2 kaylae_maxedon 62.5\n#> 3 michael_mcdole 60.9\n#> 4 devin_rivera_borrego 126. \n#> 5 james_richards 78.6\n#> 6 estela_bonilla 41.9\n#> 7 charles_najera 109. \n#> 8 reema_el_tamer 118. \n#> 9 jawhara_al_azad 90.1\n#> 10 not_applicable 84.1\n#> # ℹ 109 more rows" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#create-a-regular-grid", + "title": "3 - Tuning Hyperparameters", + "section": "Create a regular grid ", + "text": "Create a regular grid \n\nset.seed(12)\ngrid <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n grid_regular(levels = 4)\n\ngrid\n#> # A tibble: 256 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.0000000001 256 256\n#> 2 667 0.0000000001 256 256\n#> 3 1333 0.0000000001 256 256\n#> 4 2000 0.0000000001 256 256\n#> 5 1 0.0000001 256 256\n#> 6 667 0.0000001 256 256\n#> 7 1333 0.0000001 256 256\n#> 8 2000 0.0000001 256 256\n#> 9 1 0.0001 256 256\n#> 10 667 0.0001 256 256\n#> # ℹ 246 more rows" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-handling-unseen-levels", - "href": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-handling-unseen-levels", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding handling unseen levels", - "text": "Target encoding handling unseen levels\n\n\n\n\n\n\nCaution\n\n\nDon’t look at the testing data set. This is done for educational purposes.\n\n\n\n\n\n\nhotel_train |>\n count(agent, .drop = FALSE)\n#> # A tibble: 174 × 2\n#> agent n\n#> <fct> <int>\n#> 1 aaron_marquez 2\n#> 2 aayaat_al_farran 0\n#> 3 alanah_cook 0\n#> 4 alexander_drake 1117\n#> 5 allen_her 1\n#> 6 amirah_christian 0\n#> 7 anas_el_bashir 1\n#> 8 anna_beltran_moreno 0\n#> 9 anna_choi 0\n#> 10 araseli_billy 1\n#> # ℹ 164 more rows\n\n\n\nhotel_test |>\n count(agent, .drop = FALSE)\n#> # A tibble: 174 × 2\n#> agent n\n#> <fct> <int>\n#> 1 aaron_marquez 1\n#> 2 aayaat_al_farran 0\n#> 3 alanah_cook 0\n#> 4 alexander_drake 367\n#> 5 allen_her 1\n#> 6 amirah_christian 0\n#> 7 anas_el_bashir 0\n#> 8 anna_beltran_moreno 0\n#> 9 anna_choi 0\n#> 10 araseli_billy 1\n#> # ℹ 164 more rows" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-1", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-1", + "title": "3 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\n\nWhat advantage would a regular grid have?" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-handling-unseen-levels-1", - "href": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-handling-unseen-levels-1", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding handling unseen levels", - "text": "Target encoding handling unseen levels\n\n\nCalculate the global mean of the outcome and use it for cases that aren’t seen in the training data set.\n\n\nmean(hotel_train$avg_price_per_room)\n#> [1] 104.6039\n\n\n\nhotel_train |>\n summarise(\n mean = mean(avg_price_per_room),\n .by = agent\n )\n#> # A tibble: 119 × 2\n#> agent mean\n#> <fct> <dbl>\n#> 1 alexander_drake 144. \n#> 2 kaylae_maxedon 62.5\n#> 3 michael_mcdole 60.9\n#> 4 devin_rivera_borrego 126. \n#> 5 james_richards 78.6\n#> 6 estela_bonilla 41.9\n#> 7 charles_najera 109. \n#> 8 reema_el_tamer 118. \n#> 9 jawhara_al_azad 90.1\n#> 10 not_applicable 84.1\n#> # ℹ 109 more rows" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#update-parameter-ranges", + "title": "3 - Tuning Hyperparameters", + "section": "Update parameter ranges ", + "text": "Update parameter ranges \n\nlgbm_param <- \n lgbm_wflow %>% \n extract_parameter_set_dials() %>% \n update(trees = trees(c(1L, 100L)),\n learn_rate = learn_rate(c(-5, -1)))\n\nset.seed(712)\ngrid <- \n lgbm_param %>% \n grid_space_filling(size = 25)\n\ngrid\n#> # A tibble: 25 × 4\n#> trees learn_rate `agent hash` `company hash`\n#> <int> <dbl> <int> <int>\n#> 1 1 0.00147 574 574\n#> 2 5 0.00215 2048 2298\n#> 3 9 0.0000215 1824 912\n#> 4 13 0.00316 3250 512\n#> 5 17 0.0001 512 2896\n#> 6 21 0.0147 322 1625\n#> 7 25 0.1 1448 1149\n#> 8 29 0.000215 1290 256\n#> 9 34 0.0000147 456 724\n#> 10 38 0.0464 645 322\n#> # ℹ 15 more rows" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#how-do-we-handle-low-counts", - "href": "slides/advanced-04-feature-engineering-part-two.html#how-do-we-handle-low-counts", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "How do we handle low counts?", - "text": "How do we handle low counts?\n\n\nSome of the levels have very low counts. We can’t have the same confidence in those means as the means calculated on high counts.\nWe use the global mean to account for 0 occurrences. Let us adjust the calculated mean by the global mean depending on the counts.\n\n\nhotel_train |>\n summarise(\n mean = mean(avg_price_per_room),\n n = n(),\n .by = agent\n ) |>\n arrange(agent)\n#> # A tibble: 119 × 3\n#> agent mean n\n#> <fct> <dbl> <int>\n#> 1 aaron_marquez 118. 2\n#> 2 alexander_drake 144. 1117\n#> 3 allen_her 65 1\n#> 4 anas_el_bashir 99 1\n#> 5 araseli_billy 40 1\n#> 6 arhab_al_islam 35 7\n#> 7 audray_tucker 76.0 38\n#> 8 bernice_baltierra 71.3 35\n#> 9 betzy_rodriguez 84.0 66\n#> 10 brayan_guerrero 37.5 2\n#> # ℹ 109 more rows" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#the-results", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#the-results", + "title": "3 - Tuning Hyperparameters", + "section": "The results ", + "text": "The results \n\n\ngrid %>% \n ggplot(aes(trees, learn_rate)) +\n geom_point(size = 4) +\n scale_y_log10()\n\n\n\n\n\n\n\n\n\nNote that the learning rates are uniform on the log-10 scale and this shows 2 of 4 dimensions." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#partial-pooling", - "href": "slides/advanced-04-feature-engineering-part-two.html#partial-pooling", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Partial pooling", - "text": "Partial pooling\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage, though." + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choosing-tuning-parameters", + "title": "3 - Tuning Hyperparameters", + "section": "Choosing tuning parameters ", + "text": "Choosing tuning parameters \nLet’s take our previous model and tune more parameters:\n\nlgbm_spec <- \n boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(hash_rec, lgbm_spec)\n\n# Update the feature hash ranges (log-2 units)\nlgbm_param <-\n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(`agent hash` = num_hash(c(3, 8)),\n `company hash` = num_hash(c(3, 8)))" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#partial-pooling-results", - "href": "slides/advanced-04-feature-engineering-part-two.html#partial-pooling-results", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Partial pooling results", - "text": "Partial pooling results\n\n\nFigure 8" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-3", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-3", + "title": "3 - Tuning Hyperparameters", + "section": "Grid Search ", + "text": "Grid Search \n\nset.seed(9)\nctrl <- control_grid(save_pred = TRUE)\n\nlgbm_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = 25,\n # The options below are not required by default\n param_info = lgbm_param, \n control = ctrl,\n metrics = reg_metrics\n )\n\n\n\ntune_grid() is representative of tuning function syntax\nsimilar to fit_resamples()" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#implentations", - "href": "slides/advanced-04-feature-engineering-part-two.html#implentations", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Implentations", - "text": "Implentations\nWe have described this method solely based on analytical calculations (step_lencode()), but you could arrive at similar numbers using a model-based approach by fitting a no-intercept generalized linear model. A hierarchical version would induce partial pooling.\n\nstep_lencode_glm()\nstep_lencode_bayes()\nstep_lencode_mixed()" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-4", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-search-4", + "title": "3 - Tuning Hyperparameters", + "section": "Grid Search ", + "text": "Grid Search \n\nlgbm_res \n#> # Tuning results\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions \n#> <list> <chr> <list> <list> <list> \n#> 1 <split [3372/377]> Fold01 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,425 × 9]>\n#> 2 <split [3373/376]> Fold02 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 3 <split [3373/376]> Fold03 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 4 <split [3373/376]> Fold04 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 5 <split [3373/376]> Fold05 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,400 × 9]>\n#> 6 <split [3374/375]> Fold06 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,375 × 9]>\n#> 7 <split [3375/374]> Fold07 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,350 × 9]>\n#> 8 <split [3376/373]> Fold08 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 9 <split [3376/373]> Fold09 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>\n#> 10 <split [3376/373]> Fold10 <tibble [50 × 9]> <tibble [0 × 4]> <tibble [9,325 × 9]>" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-in-recipes", - "href": "slides/advanced-04-feature-engineering-part-two.html#target-encoding-in-recipes", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding in recipes ", - "text": "Target encoding in recipes \n\nrecipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_lencode(\n agent, country, company,\n outcome = vars(\"avg_price_per_room\"), smooth = TRUE,\n ) |>\n prep() |>\n bake(NULL) |>\n select(agent, country, company)\n#> # A tibble: 11,551 × 3\n#> agent country company\n#> <dbl> <dbl> <dbl>\n#> 1 144. 108. 109.\n#> 2 77.2 83.4 109.\n#> 3 61.5 99.9 109.\n#> 4 126. 108. 109.\n#> 5 144. 108. 109.\n#> 6 79.9 73.8 109.\n#> 7 126. 99.9 109.\n#> 8 69.8 108. 109.\n#> 9 126. 99.9 109.\n#> 10 144. 108. 109.\n#> # ℹ 11,541 more rows" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-results", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#grid-results", + "title": "3 - Tuning Hyperparameters", + "section": "Grid results ", + "text": "Grid results \n\nautoplot(lgbm_res)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#your-turn-2", - "href": "slides/advanced-04-feature-engineering-part-two.html#your-turn-2", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Your turn", - "text": "Your turn\n\nApply target encoding to the data set, see how it affects different predictors, not just the ones we listed here.\n\nstep_lencode()\nstep_lencode_glm()\nstep_lencode_bayes()\nstep_lencode_mixed()\n\n\n\n\n−+\n03:00" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results", + "title": "3 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(lgbm_res)\n#> # A tibble: 50 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 298 19 4.15e- 9 222 36 mae standard 53.2 10 0.427 Preprocessor01_Model1\n#> 2 298 19 4.15e- 9 222 36 rsq standard 0.810 10 0.00686 Preprocessor01_Model1\n#> 3 1394 5 5.82e- 6 28 21 mae standard 52.9 10 0.424 Preprocessor02_Model1\n#> 4 1394 5 5.82e- 6 28 21 rsq standard 0.810 10 0.00800 Preprocessor02_Model1\n#> 5 774 12 4.41e- 2 27 95 mae standard 9.77 10 0.155 Preprocessor03_Model1\n#> 6 774 12 4.41e- 2 27 95 rsq standard 0.946 10 0.00341 Preprocessor03_Model1\n#> 7 1342 7 6.84e-10 71 17 mae standard 53.2 10 0.427 Preprocessor04_Model1\n#> 8 1342 7 6.84e-10 71 17 rsq standard 0.811 10 0.00785 Preprocessor04_Model1\n#> 9 669 39 8.62e- 7 141 145 mae standard 53.2 10 0.426 Preprocessor05_Model1\n#> 10 669 39 8.62e- 7 141 145 rsq standard 0.807 10 0.00639 Preprocessor05_Model1\n#> # ℹ 40 more rows" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#date-time-variables-1", - "href": "slides/advanced-04-feature-engineering-part-two.html#date-time-variables-1", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Date time variables", - "text": "Date time variables\nHow can we represent the date column arrival_date for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model.\n\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results-1", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#tuning-results-1", + "title": "3 - Tuning Hyperparameters", + "section": "Tuning results ", + "text": "Tuning results \n\ncollect_metrics(lgbm_res, summarize = FALSE)\n#> # A tibble: 500 × 10\n#> id trees min_n learn_rate `agent hash` `company hash` .metric .estimator .estimate .config \n#> <chr> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <chr> \n#> 1 Fold01 298 19 0.00000000415 222 36 mae standard 51.8 Preprocessor01_Model1\n#> 2 Fold01 298 19 0.00000000415 222 36 rsq standard 0.821 Preprocessor01_Model1\n#> 3 Fold02 298 19 0.00000000415 222 36 mae standard 52.1 Preprocessor01_Model1\n#> 4 Fold02 298 19 0.00000000415 222 36 rsq standard 0.804 Preprocessor01_Model1\n#> 5 Fold03 298 19 0.00000000415 222 36 mae standard 52.2 Preprocessor01_Model1\n#> 6 Fold03 298 19 0.00000000415 222 36 rsq standard 0.786 Preprocessor01_Model1\n#> 7 Fold04 298 19 0.00000000415 222 36 mae standard 51.7 Preprocessor01_Model1\n#> 8 Fold04 298 19 0.00000000415 222 36 rsq standard 0.826 Preprocessor01_Model1\n#> 9 Fold05 298 19 0.00000000415 222 36 mae standard 55.2 Preprocessor01_Model1\n#> 10 Fold05 298 19 0.00000000415 222 36 rsq standard 0.845 Preprocessor01_Model1\n#> # ℹ 490 more rows" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#your-turn-3", - "href": "slides/advanced-04-feature-engineering-part-two.html#your-turn-3", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Your turn", - "text": "Your turn\n\nExplore the arrival_date variable and its relation to avg_price_per_room\nThe lubridate package might provide helpful\n\n\n\n−+\n05:00" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination", + "title": "3 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \n\nshow_best(lgbm_res, metric = \"rsq\")\n#> # A tibble: 5 × 11\n#> trees min_n learn_rate `agent hash` `company hash` .metric .estimator mean n std_err .config \n#> <int> <int> <dbl> <int> <int> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 1890 10 0.0159 115 174 rsq standard 0.948 10 0.00334 Preprocessor12_Model1\n#> 2 774 12 0.0441 27 95 rsq standard 0.946 10 0.00341 Preprocessor03_Model1\n#> 3 1638 36 0.0409 15 120 rsq standard 0.945 10 0.00384 Preprocessor16_Model1\n#> 4 963 23 0.00556 157 13 rsq standard 0.937 10 0.00320 Preprocessor06_Model1\n#> 5 590 5 0.00320 85 73 rsq standard 0.908 10 0.00465 Preprocessor24_Model1" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#arrival_date-date-features", - "href": "slides/advanced-04-feature-engineering-part-two.html#arrival_date-date-features", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "arrival_date date features", - "text": "arrival_date date features\nusing step_date( features = c(\"year\", \"month\", \"dow\", \"decimal\", \"mday\", \"doy\", \"week\", \"semester\", \"quarter\"))\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\narrival_date\nyear\nmonth\ndow\ndecimal\nmday\ndoy\nweek\nsemester\nquarter\n\n\n\n\n2016-08-30\n2016\nAug\nTue\n2016.661\n30\n243\n35\n2\n3\n\n\n2016-10-22\n2016\nOct\nSat\n2016.806\n22\n296\n43\n2\n4\n\n\n2016-12-17\n2016\nDec\nSat\n2016.959\n17\n352\n51\n2\n4\n\n\n2017-02-13\n2017\nFeb\nMon\n2017.118\n13\n44\n7\n1\n1\n\n\n2017-04-05\n2017\nApr\nWed\n2017.258\n5\n95\n14\n1\n2" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#choose-a-parameter-combination-1", + "title": "3 - Tuning Hyperparameters", + "section": "Choose a parameter combination ", + "text": "Choose a parameter combination \nCreate your own tibble for final parameters or use one of the tune::select_*() functions:\n\nlgbm_best <- select_best(lgbm_res, metric = \"mae\")\nlgbm_best\n#> # A tibble: 1 × 6\n#> trees min_n learn_rate `agent hash` `company hash` .config \n#> <int> <int> <dbl> <int> <int> <chr> \n#> 1 1890 10 0.0159 115 174 Preprocessor12_Model1" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#arrival_date-date-features-1", - "href": "slides/advanced-04-feature-engineering-part-two.html#arrival_date-date-features-1", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "arrival_date date features", - "text": "arrival_date date features\nAdding label = FALSE\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\narrival_date\nyear\nmonth\ndow\ndecimal\nmday\ndoy\nweek\nsemester\nquarter\n\n\n\n\n2016-08-30\n2016\n8\n3\n2016.661\n30\n243\n35\n2\n3\n\n\n2016-10-22\n2016\n10\n7\n2016.806\n22\n296\n43\n2\n4\n\n\n2016-12-17\n2016\n12\n7\n2016.959\n17\n352\n51\n2\n4\n\n\n2017-02-13\n2017\n2\n2\n2017.118\n13\n44\n7\n1\n1\n\n\n2017-04-05\n2017\n4\n4\n2017.258\n5\n95\n14\n1\n2" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#checking-calibration", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#checking-calibration", + "title": "3 - Tuning Hyperparameters", + "section": "Checking Calibration ", + "text": "Checking Calibration \n\n\nlibrary(probably)\nlgbm_res %>%\n collect_predictions(\n parameters = lgbm_best\n ) %>%\n cal_plot_regression(\n truth = avg_price_per_room,\n estimate = .pred\n )" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#other-recipes-steps", - "href": "slides/advanced-04-feature-engineering-part-two.html#other-recipes-steps", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Other recipes steps", - "text": "Other recipes steps\n\nstep_time()\n\nWorks the same as step_date() but for measurements smaller than day: hour, hour12, am/pm, minute, second, decimal_day.\n\nstep_holiday()\n\nAdds indicators for holidays. See timeDate::listHolidays() for supported holidays." + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel", + "title": "3 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\n\n\n\nGrid search, combined with resampling, requires fitting a lot of models!\nThese models don’t depend on one another and can be run in parallel.\n\nWe can use a parallel backend to do this:\n\ncores <- parallelly::availableCores(logical = FALSE)\ncl <- parallel::makePSOCKcluster(cores)\ndoParallel::registerDoParallel(cl)\n\n# Now call `tune_grid()`!\n\n# Shut it down with:\nforeach::registerDoSEQ()\nparallel::stopCluster(cl)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#your-turn-4", - "href": "slides/advanced-04-feature-engineering-part-two.html#your-turn-4", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Your turn", - "text": "Your turn\n\nApply date steps to the arrival_date variable and try to see if we capture anything about avg_price_per_room\n\n\n\n−+\n03:00" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#running-in-parallel-1", + "title": "3 - Tuning Hyperparameters", + "section": "Running in parallel", + "text": "Running in parallel\nSpeed-ups are fairly linear up to the number of physical cores (10 here).\n\n\nFaceted on the expensiveness of preprocessing used." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#dates-as-numerics", - "href": "slides/advanced-04-feature-engineering-part-two.html#dates-as-numerics", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "dates as numerics", - "text": "dates as numerics\n\n\nFigure 9" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#the-future-of-parallel-processing", + "title": "3 - Tuning Hyperparameters", + "section": "The ‘future’ of parallel processing", + "text": "The ‘future’ of parallel processing\nWe have relied on the foreach package for parallel processing.\nWe will start the transition to using the future package in the upcoming version of the tune package (version 1.3.0).\nThere will be a period of backward compatibility where you can still use foreach with future via the doFuture package. After that, the transition to future will occur.\nOverall, there will be minimal changes to your code." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#holidays-as-numerics", - "href": "slides/advanced-04-feature-engineering-part-two.html#holidays-as-numerics", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "holidays as numerics", - "text": "holidays as numerics\n\n\nFigure 10" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#early-stopping-for-boosted-trees", + "title": "3 - Tuning Hyperparameters", + "section": "Early stopping for boosted trees", + "text": "Early stopping for boosted trees\nWe have directly optimized the number of trees as a tuning parameter.\nInstead we could\n\nSet the number of trees to a single large number.\nStop adding trees when performance gets worse.\n\nThis is known as “early stopping” and there is a parameter for that: stop_iter.\nEarly stopping has a potential to decrease the tuning time." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#what-are-the-issues-with-these-features", - "href": "slides/advanced-04-feature-engineering-part-two.html#what-are-the-issues-with-these-features", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "What are the issues with these features?", - "text": "What are the issues with these features?\nThe numeric features make it easy to capture the end or beginning, but harder to do anything more granular.\nThe indicators mostly care about the day itself. No information about the lead-up or aftermath" + "objectID": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-2", + "href": "archive/2024-08-posit-conf/advanced-03-tuning-hyperparameters.html#your-turn-2", + "title": "3 - Tuning Hyperparameters", + "section": "Your turn", + "text": "Your turn\n\n\nSet trees = 2000 and tune the stop_iter parameter.\nNote that you will need to regenerate lgbm_param with your new workflow!\n\n\n\n−+\n10:00" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#time-events", - "href": "slides/advanced-04-feature-engineering-part-two.html#time-events", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "time events", - "text": "time events\nUsing extrasteps::step_time_event() and the almanac package, we can create useful time features.\n\nlibrary(almanac)\n\nrule_1 <- weekly() |>\n recur_on_weekdays() |>\n rsetdiff(hol_christmas())\n\nrule_2 <- monthly(since = \"2000-01-01\") |>\n recur_on_interval(3) |>\n recur_on_day_of_month(1)\n\nrule_3 <- yearly(\"1997-06-05\") |>\n recur_on_day_of_week(\"Thursday\") |>\n recur_on_month_of_year(c(\"Jun\", \"July\", \"Aug\"))" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#previously---setup", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#previously---setup", + "title": "5 - Iterative Search", + "section": "Previously - Setup", + "text": "Previously - Setup\n\n\n\nlibrary(tidymodels)\nlibrary(textrecipes)\nlibrary(bonsai)\n\n# Max's usual settings: \ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(\n pillar.advice = FALSE, \n pillar.min_title_chars = Inf\n)\n\n\n\ndata(hotel_rates)\nset.seed(295)\nhotel_rates <- \n hotel_rates %>% \n sample_n(5000) %>% \n arrange(arrival_date) %>% \n select(-arrival_date) %>% \n mutate(\n company = factor(as.character(company)),\n country = factor(as.character(country)),\n agent = factor(as.character(agent))\n )" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#step_time_event", - "href": "slides/advanced-04-feature-engineering-part-two.html#step_time_event", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "step_time_event() ", - "text": "step_time_event() \nCreate a list of rules (last slide) and pass them to the rules argument of step_time_event()\n\nrules <- list(rule_1 = rule_1, rule_2 = rule_2, rule_3 = rule_3)\n\nrecipe(~arrival_date, data = hotel_rates) |>\n step_time_event(arrival_date, rules = rules)\n\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#previously---data-usage", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#previously---data-usage", + "title": "5 - Iterative Search", + "section": "Previously - Data Usage", + "text": "Previously - Data Usage\n\nset.seed(4028)\nhotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)\n\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)\n\nset.seed(472)\nhotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#step_time_event-as-numerics", - "href": "slides/advanced-04-feature-engineering-part-two.html#step_time_event-as-numerics", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "step_time_event() as numerics ", - "text": "step_time_event() as numerics \n\n\n\n\n\n\n\n\nFigure 11" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#our-boosting-model", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#our-boosting-model", + "title": "5 - Iterative Search", + "section": "Our Boosting Model", + "text": "Our Boosting Model\nWe used feature hashing to generate a smaller set of indicator columns to deal with the large number of levels for the agent and country predictors.\n\nTree-based models (and a few others) don’t require indicators for categorical predictors. They can split on these variables as-is.\n\nWe’ll keep all categorical predictors as factors and focus on optimizing additional boosting parameters." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#non-indicator-time-events", - "href": "slides/advanced-04-feature-engineering-part-two.html#non-indicator-time-events", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Non-indicator time events", - "text": "Non-indicator time events\nThese features still have the issue that they only attach value to the date itself.\nWe can attach values based on how far we are away from those dates.\n\nstep_date_before()\nstep_date_after()\nstep_date_nearest()" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#our-boosting-model-1", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#our-boosting-model-1", + "title": "5 - Iterative Search", + "section": "Our Boosting Model", + "text": "Our Boosting Model\n\nlgbm_spec <- \n boost_tree(trees = 1000, learn_rate = tune(), min_n = tune(), \n tree_depth = tune(), loss_reduction = tune(), \n stop_iter = tune()) %>% \n set_mode(\"regression\") %>% \n set_engine(\"lightgbm\", num_threads = 1)\n\nlgbm_wflow <- workflow(avg_price_per_room ~ ., lgbm_spec)\n\nlgbm_param <- \n lgbm_wflow %>%\n extract_parameter_set_dials() %>%\n update(learn_rate = learn_rate(c(-5, -1)))" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#step_date_before", - "href": "slides/advanced-04-feature-engineering-part-two.html#step_date_before", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "step_date_before()", - "text": "step_date_before()\n\n\nFigure 12" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#iterative-search", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#iterative-search", + "title": "5 - Iterative Search", + "section": "Iterative Search", + "text": "Iterative Search\nInstead of pre-defining a grid of candidate points, we can model our current results to predict what the next candidate point should be.\n\nSuppose that we are only tuning the learning rate in our boosted tree.\n\nWe could do something like:\nmae_pred <- lm(mae ~ learn_rate, data = resample_results)\nand use this to predict and rank new learning rate candidates." }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#step_date_before---inverse", - "href": "slides/advanced-04-feature-engineering-part-two.html#step_date_before---inverse", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "step_date_before() - inverse", - "text": "step_date_before() - inverse\n\n\nFigure 13" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#iterative-search-1", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#iterative-search-1", + "title": "5 - Iterative Search", + "section": "Iterative Search", + "text": "Iterative Search\nA linear model probably isn’t the best choice though (more in a minute).\nTo illustrate the process, we resampled a large grid of learning rate values for our data to show what the relationship is between MAE and learning rate.\nNow suppose that we used a grid of three points in the parameter range for learning rate…" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#step_date_after---inverse", - "href": "slides/advanced-04-feature-engineering-part-two.html#step_date_after---inverse", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "step_date_after() - inverse", - "text": "step_date_after() - inverse\n\n\nFigure 14" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#a-large-grid", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#a-large-grid", + "title": "5 - Iterative Search", + "section": "A Large Grid", + "text": "A Large Grid" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#step_date_nearest---inverse", - "href": "slides/advanced-04-feature-engineering-part-two.html#step_date_nearest---inverse", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "step_date_nearest() - inverse", - "text": "step_date_nearest() - inverse\n\n\nFigure 15" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#a-three-point-grid", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#a-three-point-grid", + "title": "5 - Iterative Search", + "section": "A Three Point Grid", + "text": "A Three Point Grid" }, { - "objectID": "slides/advanced-04-feature-engineering-part-two.html#datetime-features", - "href": "slides/advanced-04-feature-engineering-part-two.html#datetime-features", - "title": "6 - Feature engineering: splines, target encoding and dates", - "section": "Datetime features", - "text": "Datetime features\nAvoid crafting datetime features by hand if at all possible.\nDealing with uneven month lengths, leap days (leap seconds)\nOr tried to define any event that doesn’t land on the same day of the week or date each year.\n\nThe first Sunday after the first full moon on or after the vernal equinox" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#gaussian-processes-and-optimization", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#gaussian-processes-and-optimization", + "title": "5 - Iterative Search", + "section": "Gaussian Processes and Optimization", + "text": "Gaussian Processes and Optimization\nWe can make a “meta-model” with a small set of historical performance results.\nGaussian Processes (GP) models are a good choice to model performance.\n\nIt is a Bayesian model so we are using Bayesian Optimization (BO).\nFor regression, we can assume that our data are multivariate normal.\nWe also define a covariance function for the variance relationship between data points. A common one is:\n\n\\[\\operatorname{cov}(\\boldsymbol{x}_i, \\boldsymbol{x}_j) = \\exp\\left(-\\frac{1}{2}|\\boldsymbol{x}_i - \\boldsymbol{x}_j|^2\\right) + \\sigma^2_{ij}\\]\n\nGPs are good because\n\nthey are flexible regression models (in the sense that splines are flexible).\nwe need to get mean and variance predictions (and they are Bayesian)\ntheir variability is based on spatial distances.\n\nSome people use random forests (with conformal variance estimates) or other methods but GPs are most popular." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#getting-set-up", - "href": "slides/advanced-05-feature-engineering-part-one.html#getting-set-up", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Getting set up ", - "text": "Getting set up \n\nlibrary(tidymodels)\nlibrary(embed)\nlibrary(extrasteps)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)\n\n# Load our example data for this section\n\"https://github.com/tidymodels/workshops/raw/refs/heads/2025-GMOFETML/slides/leaf_data.RData\" |> \n url() |> \n load()" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#predicting-candidates", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#predicting-candidates", + "title": "5 - Iterative Search", + "section": "Predicting Candidates", + "text": "Predicting Candidates\nThe GP model can take candidate tuning parameter combinations as inputs and make predictions for performance (e.g. MAE)\n\nThe mean performance\nThe variance of performance\n\nThe variance is mostly driven by spatial variability (the previous equation).\nThe predicted variance is zero at locations of actual data points and becomes very high when far away from any observed data." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#leaf-data-set", - "href": "slides/advanced-05-feature-engineering-part-one.html#leaf-data-set", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Leaf data set", - "text": "Leaf data set\nSlightly modified version of modeldata::leaf_id_flavia.\n\nglimpse(leaf_data)\n#> Rows: 1,907\n#> Columns: 55\n#> $ species <fct> chinese_redbud, chinese_redbud, chinese_red…\n#> $ apex <fct> none, none, none, none, none, none, none, n…\n#> $ base <fct> none, none, none, none, none, none, none, n…\n#> $ shape <fct> heart_shape, heart_shape, heart_shape, hear…\n#> $ edge <chr> \"smooth\", \"smooth\", \"smooth\", \"smooth\", \"sm…\n#> $ outlying_polar <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ skewed_polar <dbl> 0.4610066, 0.4747877, 0.5356883, 0.4771338,…\n#> $ clumpy_polar <dbl> 0.006479784, 0.011100482, 0.017317486, 0.01…\n#> $ sparse_polar <dbl> 0.01449941, 0.01451566, 0.02953578, 0.01425…\n#> $ striated_polar <dbl> 0.9788360, 0.9797980, 0.6758621, 0.9896373,…\n#> $ convex_polar <dbl> 0.000899987, 0.000152532, 0.035230741, 0.00…\n#> $ skinny_polar <dbl> 0.1177954, 0.5440493, 0.7764908, 0.7067394,…\n#> $ stringy_polar <dbl> 1.0000000, 1.0000000, 0.8544140, 1.0000000,…\n#> $ monotonic_polar <dbl> 0.026807610, 0.005554220, 0.068481538, 0.09…\n#> $ outlying_contour <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ skewed_contour <dbl> 0.4954035, 0.5321818, 0.5451171, 0.4675013,…\n#> $ clumpy_contour <dbl> 0.007898114, 0.010846498, 0.020271493, 0.01…\n#> $ sparse_contour <dbl> 0.01449941, 0.01436114, 0.02903878, 0.01451…\n#> $ striated_contour <dbl> 0.9744898, 0.9811321, 0.6766917, 0.9784946,…\n#> $ convex_contour <dbl> 0.000479470, 0.000000000, 0.035908221, 0.00…\n#> $ skinny_contour <dbl> 0.1810093, 1.0000000, 0.7806988, 0.1348225,…\n#> $ stringy_contour <dbl> 1.0000000, 1.0000000, 0.8864787, 1.0000000,…\n#> $ monotonic_contour <dbl> 0.000816260, 0.000008740, 0.001253195, 0.32…\n#> $ num_max_points <dbl> 1, 3, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 6, 1…\n#> $ num_min_points <dbl> 2, 4, 7, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 8, 2…\n#> $ diameter <dbl> 1255.830, 1227.826, 1113.228, 1219.185, 118…\n#> $ area <dbl> 936626.5, 917300.5, 855376.5, 901303.0, 885…\n#> $ perimeter <dbl> 3830.534, 3781.445, 3695.646, 3678.409, 383…\n#> $ physiological_length <dbl> 1253, 1218, 1098, 1205, 1153, 1179, 1188, 1…\n#> $ physiological_width <dbl> 1088, 1091, 1083, 1053, 1078, 1073, 1054, 1…\n#> $ aspect_ratio <dbl> 0.8683160, 0.8957307, 0.9863388, 0.8738589,…\n#> $ rectangularity <dbl> 0.6870470, 0.6903027, 0.7193273, 0.7103222,…\n#> $ circularity <dbl> 0.8021540, 0.8061315, 0.7870211, 0.8370680,…\n#> $ compactness <dbl> 15.66578, 15.58849, 15.96701, 15.01237, 16.…\n#> $ narrow_factor <dbl> 1.154255, 1.125413, 1.027912, 1.157821, 1.1…\n#> $ perimeter_ratio_diameter <dbl> 3.050202, 3.079790, 3.319756, 3.017104, 3.2…\n#> $ perimeter_ratio_length <dbl> 3.520711, 3.466036, 3.412416, 3.493266, 3.5…\n#> $ perimeter_ratio_lw <dbl> 1.636281, 1.637698, 1.694473, 1.629056, 1.7…\n#> $ num_convex_points <dbl> 125, 128, 114, 138, 125, 125, 121, 112, 115…\n#> $ perimeter_convexity <dbl> 0.9404967, 0.9393550, 0.9232549, 0.9502414,…\n#> $ area_convexity <dbl> 0.02359372, 0.03175513, 0.03932713, 0.01740…\n#> $ area_ratio_convexity <dbl> 0.9769501, 0.9692222, 0.9621610, 0.9828892,…\n#> $ equivalent_diameter <dbl> 1092.0393, 1080.7142, 1043.5992, 1071.2491,…\n#> $ eccentricity <dbl> 0.44400376, 0.26866788, 0.31952502, 0.44811…\n#> $ contrast <dbl> 38.69163, 32.73154, 23.20849, 30.14546, 23.…\n#> $ correlation_texture <dbl> 0.9959609, 0.9967740, 0.9975354, 0.9968981,…\n#> $ inverse_difference_moments <dbl> 0.5951133, 0.6022763, 0.6363508, 0.6135506,…\n#> $ entropy <dbl> 6.291552, 6.273267, 5.764995, 6.203471, 5.9…\n#> $ mean_red_val <dbl> 38.02429, 34.55425, 30.92509, 33.44151, 33.…\n#> $ mean_green_val <dbl> 72.38449, 69.89448, 69.22250, 71.32743, 70.…\n#> $ mean_blue_val <dbl> 42.78902, 42.93044, 40.80089, 40.73282, 42.…\n#> $ std_red_val <dbl> 41.66486, 40.48649, 38.55110, 39.09408, 40.…\n#> $ std_green_val <dbl> 74.26665, 73.05275, 76.77596, 75.85529, 75.…\n#> $ std_blue_val <dbl> 46.36232, 48.04195, 48.09177, 46.57170, 48.…\n#> $ correlation <dbl> -0.027629688, -0.003103965, -0.036341630, 0…" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#your-turn", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#your-turn", + "title": "5 - Iterative Search", + "section": "Your turn", + "text": "Your turn\n\n\nYour GP makes predictions on two new candidate tuning parameters.\nWe want to minimize MAE.\nWhich should we choose?\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n−+\n03:00" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#leaf-data-set-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#leaf-data-set-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Leaf data set", - "text": "Leaf data set\n\n\nLakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077." + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-fit-ribbon-is-mean---1sd", + "title": "5 - Iterative Search", + "section": "GP Fit (ribbon is mean +/- 1SD)", + "text": "GP Fit (ribbon is mean +/- 1SD)" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#leaf-data-set-2", - "href": "slides/advanced-05-feature-engineering-part-one.html#leaf-data-set-2", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Leaf data set", - "text": "Leaf data set\n\n\nLakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077." + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#choosing-new-candidates", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#choosing-new-candidates", + "title": "5 - Iterative Search", + "section": "Choosing New Candidates", + "text": "Choosing New Candidates\nThis isn’t a very good fit but we can still use it.\nHow can we use the outputs to choose the next point to measure?\n\nAcquisition functions take the predicted mean and variance and use them to balance:\n\nexploration: new candidates should explore new areas.\nexploitation: new candidates must stay near existing values.\n\nExploration focuses on the variance, exploitation is about the mean." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#leaf-data-set-3", - "href": "slides/advanced-05-feature-engineering-part-one.html#leaf-data-set-3", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Leaf data set", - "text": "Leaf data set\n\n\nLakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077." + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#acquisition-functions", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#acquisition-functions", + "title": "5 - Iterative Search", + "section": "Acquisition Functions", + "text": "Acquisition Functions\nWe’ll use an acquisition function to select a new candidate.\nThe most popular method appears to be expected improvement (EI) above the current best results.\n\nZero at existing data points.\nThe expected improvement is integrated over all possible improvement (“expected” in the probability sense).\n\nWe would probably pick the point with the largest EI as the next point.\n(There are other functions beyond EI.)" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#your-turn", - "href": "slides/advanced-05-feature-engineering-part-one.html#your-turn", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Your turn", - "text": "Your turn\n\nLoad and explore the leaf data\nMake time to look at the edge column and think about how one would encode it into numerics\n\n\n\n−+\n05:00" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement", + "title": "5 - Iterative Search", + "section": "Expected Improvement", + "text": "Expected Improvement" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#leaf-edges", - "href": "slides/advanced-05-feature-engineering-part-one.html#leaf-edges", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Leaf edges", - "text": "Leaf edges\n\n\n\nLow number of unique levels\nSome overlap\n\n\n\nleaf_data |>\n count(edge)\n#> # A tibble: 7 × 2\n#> edge n\n#> <chr> <int>\n#> 1 \"\" 62\n#> 2 \"denate\" 65\n#> 3 \"lobed\" 109\n#> 4 \"lobed, smooth\" 52\n#> 5 \"lobed, toothed\" 109\n#> 6 \"smooth\" 1204\n#> 7 \"toothed\" 306" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#iteration", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#iteration", + "title": "5 - Iterative Search", + "section": "Iteration", + "text": "Iteration\nOnce we pick the candidate point, we measure performance for it (e.g. resampling).\n\nAnother GP is fit, EI is recomputed, and so on.\n\nWe stop when we have completed the allowed number of iterations or if we don’t see any improvement after a pre-set number of attempts." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#leaf-edges-dummies", - "href": "slides/advanced-05-feature-engineering-part-one.html#leaf-edges-dummies", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Leaf edges dummies", - "text": "Leaf edges dummies\n(technically one-hot encoding)\n\n\n\n\n\nX\ndenate\nlobed\nlobed..smooth\nlobed..toothed\nsmooth\ntoothed\n\n\n\n\n0\n0\n0\n0\n1\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n1\n\n\n0\n0\n0\n0\n0\n1\n0\n\n\n0\n0\n0\n0\n0\n1\n0\n\n\n0\n0\n0\n0\n0\n0\n1\n\n\n0\n0\n1\n0\n0\n0\n0" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-fit-with-four-points", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-fit-with-four-points", + "title": "5 - Iterative Search", + "section": "GP Fit with four points", + "text": "GP Fit with four points" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#dummy-variables", - "href": "slides/advanced-05-feature-engineering-part-one.html#dummy-variables", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Dummy variables", - "text": "Dummy variables\n\n\nPros\n\nCommonly used\nEasy interpretation\nWill rarely lead to a decrease in performance\n\n\nCons\n\nCan create many columns\nNeeds clean levels\nNot likely the most efficient representation" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement-1", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement-1", + "title": "5 - Iterative Search", + "section": "Expected Improvement", + "text": "Expected Improvement" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#overlapping", - "href": "slides/advanced-05-feature-engineering-part-one.html#overlapping", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Overlapping", - "text": "Overlapping\n\n\nSome of the labels\n\nlobed\nsmooth\ntoothed\nlobed, smooth\nlobed, toothed\n\n\nIf you want to find a “toothed” leaf, which level do you pick?\n\ntoothed\nlobed, toothed\ntoothed and lobed, toothed\n\n\nWe can let lobed, toothed be counted for lobed and toothed" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-evolution", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#gp-evolution", + "title": "5 - Iterative Search", + "section": "GP Evolution", + "text": "GP Evolution" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies", - "text": "Advanced Dummies\nThis is basically a poor man’s Natural Language Processing.\n\n\ntokenization -> counting\n\n\nWe think it is a frequent enough case that it is considered its own method.\nWe have 2 variants: “extraction” and “multi choice”" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement-evolution", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#expected-improvement-evolution", + "title": "5 - Iterative Search", + "section": "Expected Improvement Evolution", + "text": "Expected Improvement Evolution" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction", - "href": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\nWorks on a singular column, using a regular expression to extract the items we want to count.\nDone using regular expressions, either by specifying sep to split the string by, or by using pattern to extract the items.\nAllows for 0 to many items in each string.\nImplemented as step_dummy_extract().\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#bo-in-tidymodels", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#bo-in-tidymodels", + "title": "5 - Iterative Search", + "section": "BO in tidymodels", + "text": "BO in tidymodels\nWe’ll use a function called tune_bayes() that has very similar syntax to tune_grid().\n\nIt has an additional initial argument for the initial set of performance estimates and parameter combinations for the GP model." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"\"\n\"denate\"\n\"lobed\"\n\"lobed, smooth\"\n\"lobed, toothed\"\n\"smooth\"\n\"toothed\"\n\n\nUsing\nsep = \", \"\n\nResult\n\n\n\n\n\ndenate\nlobed\nsmooth\ntoothed\n\n\n\n\n0\n0\n0\n0\n\n\n1\n0\n0\n0\n\n\n0\n1\n0\n0\n\n\n0\n1\n1\n0\n\n\n0\n1\n0\n1\n\n\n0\n0\n1\n0\n\n\n0\n0\n0\n1" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#initial-grid-points", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#initial-grid-points", + "title": "5 - Iterative Search", + "section": "Initial grid points", + "text": "Initial grid points\ninitial can be the results of another tune_*() function or an integer (in which case tune_grid() is used under to hood to make such an initial set of results).\n\nWe’ll run the optimization more than once, so let’s make an initial grid of results to serve as the substrate for the BO.\nI suggest at least the number of tuning parameters plus two as the initial grid for BO." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction-2", - "href": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction-2", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"Etching on paper\"\n\"Oil paint on canvas\"\n\"Acrylic paint on paper\"\n\"Oil paint and wax on canvas\"\n\"Oil paint, ink on canvas\"\n\n\nUsing\nsep = \", \"\n\nResult\n\n\n\n\n\n\n\n\n\n\n\n\n\nAcrylic.paint.on.paper\nEtching.on.paper\nink.on.canvas\nOil.paint\nOil.paint.and.wax.on.canvas\nOil.paint.on.canvas\n\n\n\n\n0\n1\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n1\n\n\n1\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n1\n0\n\n\n0\n0\n1\n1\n0\n0" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#an-initial-grid", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#an-initial-grid", + "title": "5 - Iterative Search", + "section": "An Initial Grid", + "text": "An Initial Grid\n\nreg_metrics <- metric_set(mae, rsq)\n\nset.seed(12)\ninit_res <-\n lgbm_wflow %>%\n tune_grid(\n resamples = hotel_rs,\n grid = nrow(lgbm_param) + 2,\n param_info = lgbm_param,\n metrics = reg_metrics\n )\n\nshow_best(init_res, metric = \"mae\") %>% select(-.metric, -.estimator)\n#> # A tibble: 5 × 9\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config \n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> \n#> 1 16 12 0.0136 1.91e- 3 9 10.1 10 0.196 Preprocessor1_Model4\n#> 2 9 4 0.0415 5.21e- 9 13 10.2 10 0.167 Preprocessor1_Model1\n#> 3 25 8 0.00256 9.58e-10 7 14.1 10 0.278 Preprocessor1_Model7\n#> 4 22 9 0.00154 5.77e- 6 5 19.3 10 0.326 Preprocessor1_Model5\n#> 5 32 3 0.000144 3.02e+ 1 18 47.6 10 0.387 Preprocessor1_Model6" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction-3", - "href": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction-3", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"Etching on paper\"\n\"Oil paint on canvas\"\n\"Acrylic paint on paper\"\n\"Oil paint and wax on canvas\"\n\"Oil paint, ink on canvas\"\n\n\nUsing\nsep = \"(, )|( and )|( on )\"\n\nResult\n\n\n\n\n\nAcrylic.paint\ncanvas\nEtching\nink\nOil.paint\npaper\nwax\n\n\n\n\n0\n0\n1\n0\n0\n1\n0\n\n\n0\n1\n0\n0\n1\n0\n0\n\n\n1\n0\n0\n0\n0\n1\n0\n\n\n0\n1\n0\n0\n1\n0\n1\n\n\n0\n1\n0\n1\n1\n0\n0" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#bo-using-tidymodels", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#bo-using-tidymodels", + "title": "5 - Iterative Search", + "section": "BO using tidymodels", + "text": "BO using tidymodels\n\nctrl_bo <- control_bayes(verbose_iter = TRUE) # <- for demonstration\n\nset.seed(15)\nlgbm_bayes_res <-\n lgbm_wflow %>%\n tune_bayes(\n resamples = hotel_rs,\n initial = init_res, # <- initial results\n iter = 20,\n param_info = lgbm_param,\n control = ctrl_bo,\n metrics = reg_metrics\n )\n#> Optimizing mae using the expected improvement\n#> \n#> ── Iteration 1 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=10.13 (@iter 0)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=32, tree_depth=12, learn_rate=0.0178, loss_reduction=1.03e-10, stop_iter=12\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=10.08 (+/-0.175)\n#> \n#> ── Iteration 2 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=10.08 (@iter 1)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=15, tree_depth=14, learn_rate=0.0977, loss_reduction=0.00535, stop_iter=4\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.719 (+/-0.187)\n#> \n#> ── Iteration 3 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=38, tree_depth=1, learn_rate=0.1, loss_reduction=0.0809, stop_iter=10\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=15.45 (+/-0.253)\n#> \n#> ── Iteration 4 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=32, tree_depth=1, learn_rate=0.00833, loss_reduction=1.31e-06, stop_iter=8\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=19.44 (+/-0.33)\n#> \n#> ── Iteration 5 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=18, tree_depth=8, learn_rate=0.0495, loss_reduction=1.4e-06, stop_iter=5\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.757 (+/-0.146)\n#> \n#> ── Iteration 6 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=3, tree_depth=14, learn_rate=0.0319, loss_reduction=4.02e-09, stop_iter=17\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.76 (+/-0.163)\n#> \n#> ── Iteration 7 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.719 (@iter 2)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=6, tree_depth=8, learn_rate=0.0883, loss_reduction=1.94e-08, stop_iter=4\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.712 (+/-0.17)\n#> \n#> ── Iteration 8 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.712 (@iter 7)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=6, tree_depth=8, learn_rate=0.025, loss_reduction=7.82e-05, stop_iter=19\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.838 (+/-0.17)\n#> \n#> ── Iteration 9 ───────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.712 (@iter 7)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=32, tree_depth=6, learn_rate=0.0737, loss_reduction=2.15e-07, stop_iter=8\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.06 (+/-0.2)\n#> \n#> ── Iteration 10 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.712 (@iter 7)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=5, tree_depth=11, learn_rate=0.0451, loss_reduction=3.45e-10, stop_iter=7\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.637 (+/-0.156)\n#> \n#> ── Iteration 11 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=7, learn_rate=0.0372, loss_reduction=2.44e-09, stop_iter=11\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.761 (+/-0.171)\n#> \n#> ── Iteration 12 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=26, tree_depth=15, learn_rate=0.00626, loss_reduction=0.00554, stop_iter=16\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=10.79 (+/-0.198)\n#> \n#> ── Iteration 13 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=29, tree_depth=10, learn_rate=0.0996, loss_reduction=4.5e-05, stop_iter=16\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.838 (+/-0.169)\n#> \n#> ── Iteration 14 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=12, tree_depth=13, learn_rate=0.085, loss_reduction=2.16, stop_iter=9\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.795 (+/-0.16)\n#> \n#> ── Iteration 15 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=4, tree_depth=9, learn_rate=0.0418, loss_reduction=0.00293, stop_iter=7\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.75 (+/-0.168)\n#> \n#> ── Iteration 16 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=6, tree_depth=15, learn_rate=0.0703, loss_reduction=5.15e-10, stop_iter=13\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.672 (+/-0.134)\n#> \n#> ── Iteration 17 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=27, tree_depth=15, learn_rate=0.0956, loss_reduction=3.74e-10, stop_iter=17\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.861 (+/-0.197)\n#> \n#> ── Iteration 18 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.637 (@iter 10)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=11, learn_rate=0.0871, loss_reduction=0.00196, stop_iter=18\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results: mae=9.601 (+/-0.147)\n#> \n#> ── Iteration 19 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.601 (@iter 18)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=2, tree_depth=12, learn_rate=0.0991, loss_reduction=8.45e-06, stop_iter=14\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.61 (+/-0.17)\n#> \n#> ── Iteration 20 ──────────────────────────────────────────────────────\n#> \n#> i Current best: mae=9.601 (@iter 18)\n#> i Gaussian process model\n#> ✓ Gaussian process model\n#> i Generating 5000 candidates\n#> i Predicted candidates\n#> i min_n=4, tree_depth=15, learn_rate=0.0206, loss_reduction=1.46e-06, stop_iter=15\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results: mae=9.881 (+/-0.177)" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction-4", - "href": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---extraction-4", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"['red', 'blue']\"\n\"['red', 'blue', 'white']\"\n\"['blue', 'blue', 'blue']\"\n\n\nUsing\npattern = \"(?<=')[^',]+(?=')\"\n\nResult\n\n\n\n\n\nblue\nred\nwhite\n\n\n\n\n1\n1\n0\n\n\n1\n1\n1\n\n\n3\n0\n0" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#best-results", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#best-results", + "title": "5 - Iterative Search", + "section": "Best results", + "text": "Best results\n\nshow_best(lgbm_bayes_res, metric = \"mae\") %>% select(-.metric, -.estimator)\n#> # A tibble: 5 × 10\n#> min_n tree_depth learn_rate loss_reduction stop_iter mean n std_err .config .iter\n#> <int> <int> <dbl> <dbl> <int> <dbl> <int> <dbl> <chr> <int>\n#> 1 2 11 0.0871 1.96e- 3 18 9.60 10 0.147 Iter18 18\n#> 2 2 12 0.0991 8.45e- 6 14 9.61 10 0.170 Iter19 19\n#> 3 5 11 0.0451 3.45e-10 7 9.64 10 0.156 Iter10 10\n#> 4 6 15 0.0703 5.15e-10 13 9.67 10 0.134 Iter16 16\n#> 5 6 8 0.0883 1.94e- 8 4 9.71 10 0.170 Iter7 7" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---multi-choice", - "href": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---multi-choice", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Multi Choice", - "text": "Advanced Dummies - Multi Choice\nWorks on multiple columns, counting items across columns.\nIt can be seen as joining multiple applications of dummy variables together.\nAllows for 0 to many items.\nImplemented as step_dummy_multi_choice().\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results", + "title": "5 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\")" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---multi-choice-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#advanced-dummies---multi-choice-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Multi Choice", - "text": "Advanced Dummies - Multi Choice\n\n\nInput\n\n\n\n\n\nlang_1\nlang_2\nlang_3\n\n\n\n\nEnglish\nNA\nItalian\n\n\nSpanish\nFrench\nFrench\n\n\nNA\nNA\nNA\n\n\nEnglish\nNA\nNA\n\n\n\n\n\n\nResult\n\n\n\n\n\nEnglish\nFrench\nItalian\nSpanish\n\n\n\n\n1\n0\n1\n0\n\n\n0\n1\n0\n1\n\n\n0\n0\n0\n0\n\n\n1\n0\n0\n0" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results-1", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results-1", + "title": "5 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"parameters\")" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#othering", - "href": "slides/advanced-05-feature-engineering-part-one.html#othering", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Othering", - "text": "Othering\nBoth step_dummy_extract() and step_dummy_multi_choice() contain a threshold argument.\nThis is used to combine infrequent levels together.\nWe have a step step_other() that does this for nominal variables, but it doesn’t work in these cases, as it has to happen after the extraction/combination." + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results-2", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#plotting-bo-results-2", + "title": "5 - Iterative Search", + "section": "Plotting BO Results", + "text": "Plotting BO Results\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\")" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#othering-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#othering-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Othering", - "text": "Othering\nthreshold = 0 produces 1217 columns.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nX1.monitor\nX1.person\nX1.projection\nX10.light.boxes\nX10.tranformers\nX100.digital.prints\nX100.works\nX11.photographs\nX11.works\nX12.ceramic.tea.mugs\nX12.drawings\nX13.canvases\nX133.slides\nX14.monitors\nX14.photographs\nX14.small.cubes\nX14.works\nX15.engraved.plaques\nX15.hand.coloured.photographs\nX15.photographs\nX15.rooms\nX15.steel.kitchen.ustensils\nX15.unglazed.ceramic.forms\nX151.text.panels\nX16.channels\nX16.mm\nX16.paintings\nX16.sheets\nX160.slides\nX16mm\nX18.drawings\nX18.photographs\nX19.wooden.pieces.of.furniture\nX2.aluminium.panels\nX2.aluminium.tables\nX2.back.projections\nX2.banners\nX2.canvases\nX2.car.batteries\nX2.copper.panels\nX2.digital.prints\nX2.electric.cables\nX2.flat.screens\nX2.fluorescent.tubes\nX2.hooks\nX2.horses\nX2.light.bulbs\nX2.lithographs\nX2.maps\nX2.marker.pens\nX2.metal.scaffold.towers\nX2.monitors\nX2.paintings\nX2.papers\nX2.people\nX2.photographs\nX2.porcelain.light.sockets\nX2.projections\nX2.ropes\nX2.screenprints\nX2.sections.of.pre.cast.concrete.pipe\nX2.sheets\nX2.slippers\nX2.stainless.steel.discs\nX2.trains\nX2.transparencies\nX2.videos\nX2.wall.pieces\nX2.works\nX20.cotton.mattresses\nX20.flat.screens.or.1.projection\nX20.photographs\nX20.wooden.beds\nX2000.gerberas\nX201.photographs\nX21.aluminium.bricks\nX21.channels\nX21.works\nX22.photographs\nX220.works\nX23.monitors\nX24.light.bulbs\nX24.works\nX25.photographs\nX26.works\nX27.photographs\nX28.etchings\nX3.balls\nX3.books\nX3.canvases\nX3.flat.screens\nX3.Kentia.palms.in.3.terracotta.pots\nX3.lamps\nX3.light.bulbs\nX3.monitors\nX3.people\nX3.photographs\nX3.printers\nX3.prints\nX3.projections\nX3.screenprints\nX3.skateboards\nX3.transformers\nX3.wall.mounted.LCD.monitors\nX3.works\nX30.light.emitting.diodes.units\nX30.works\nX31.photographs\nX32.digital.prints\nX32.flat.screens\nX32.papers\nX320.slides\nX34.photographs\nX34.wooden.sculptures\nX35.mm\nX35.works\nX350.digital.prints\nX355.photographs\nX35mm\nX36.tin.dogs\nX39.cardboard.boxes\nX39.metronomes\nX4.aluminium.pixel.boxes.with.DMX.control.box\nX4.Belisha.beacons\nX4.canvases\nX4.channels\nX4.corrugated.slabs.of.concrete\nX4.digital.prints\nX4.digital.prints.with.acrylic.paint\nX4.fibreboard.panels\nX4.flat.screens\nX4.gascookers\nX4.hardboard.walls\nX4.light.bulbs\nX4.mannequins\nX4.monitors\nX4.photocopies\nX4.photographs\nX4.projections\nX4.steel.capsules\nX40.light.emitting.diode.units\nX40.photographs\nX40.works\nX42.electric.lamps\nX42.photographs\nX46.photographs\nX49.Perspex.boxes\nX5.digital.prints\nX5.drawings\nX5.flat.screens\nX5.knives\nX5.monitors\nX5.painted.plaster.busts\nX5.painted.wooden.shelves\nX5.photographs\nX5.projections\nX5.tables\nX5.works\nX50.canvases\nX50.cardboard.boxes\nX50.slides\nX53.photographs\nX54.digital.prints\nX6.digital.prints\nX6.fabrics\nX6.lithographs\nX6.monitors\nX6.Perspex.panels\nX6.photographs\nX6.projections\nX6.works\nX60.digital.prints\nX60.works\nX62.florescent.lights\nX64.tin.soldiers\nX7.channels\nX7.drawings\nX7.light.emitting.diode.columns\nX7.panels\nX7.projections\nX7.stools\nX7.works\nX71.vinyl.records\nX76.works\nX8.channels\nX8.digital.prints\nX8.headphones\nX8.letterpress.prints\nX8.mm\nX8.monitors\nX8.projections\nX8.synthetic.fabric.flags\nX8.woodblock\nX8.works\nX80.slides\nX800.digital.prints\nX84.sleeves\nX9.doors\nX9.LED.lights\nX9.photographs\nX9.works\nX90.photographs\nacetate\nacrylic\nacrylic.fibre\nacrylic.glass\nacrylic.pai\nacrylic.paint\nAcrylic.paint\nacrylic.resin\nacrylic.sheet\nAcrylic.sheet\nacrylic.sheets\nAcrylic.tubes\nadhesive\nadhesive.tape\naerials\nAgfa.Isolette.camera\nAlabaster\nalarm.clocks\nAlkyd.paint\nAltered.Eames.plywood.leg.splints\naluminium\nAluminium\naluminium.buckles\naluminium.disc\naluminium.lightbox\nAluminium.machinery.part\naluminium.paint\naluminium.panel\naluminuim\nAluminuim\naluminum.panel\namplifier\nand.sound\nand.sound..mono\nand.sound..mono.\nand.sound..stereo.\nanimal.bones\nAnimal.intestines\naquatint\nAquatint\naquatint.with.hand.colouring\nartificial.foliage\nartificial.moss\nartificial.sand\nartificial.wig\nartist.s.hair\nash\nAsh\nashtray\naudio\nAudio\naudio.system\naudiotrack\nback.projection\nbadge\nbadges.with.printed.papers\nballasts\nbalsa.wood\nBamboo.cane\nBamboo.poles\nbark\nbaseball.cap\nbath.bombs\nbatteri\nbauxite.rocks\nbeads\nBeanbag\nbeech\nbeer.can\nbells\nbicycle.belt\nbindis\nbins\nblack\nblackboard\nBlind.embossed.print\nboard\nBoard\nBoat\nbone\nbones\nbook\nBook\nbook.cover\nbooklets\nbooks\nboots\nbowl\nbox\nbrass\nBrass\nbrass.pins\nbrass.plate\nbreeze.blocks\nbrick\nBriefcase\nbronze\nBronze\nBronze.bells\nbronze.powder\nBronze.with.silk.scarf\nBronze..with.Jamie.Sargeant\nBronze..with.Nicholas.Sloan\nbrooms\nbuckets\nbuckram\nbuoy\nburlaq.sack\nburnt.wood\nButterflies\nbuttons\ncable\ncables\ncalf\ncalico\ncandle\ncanvas\nCanvas\nCanvas.lining.with.ingrained.dust\ncanvas.paper\nCanvas.tacking.edges\ncarbon.fibre\ncarborundum\ncarborundum.mezzotint\ncard\nCard\ncardboard\nCardboard\ncardboard.architectural.model\ncardboard.base\ncardboard.box\nCardboard.box\ncardboard.boxes\ncardboard.coffin\ncarpet\nCarpet\ncarpets\nCarrara.marble\nCast.iron\ncastor.wheels\nCedar.wood\nceiling\nCellophane\ncellulose\ncellulose.lacquer\ncellulose.print\ncement\ncement.blocks\ncentral.processing.unit\nceramic\nCeramic\nceramic.tiles\nchain\nchair\nchair.parts\nchairs\nchalk\nChalk\ncharcoal\nCharcoal\nchine.collé\nchipboard\nChipboard\nchrome\nchrome.steel.balls\nChromogenic.print\ncibachrome\ncibachrome.print\nCibachrome.print\ncigarette.ash\ncigarette.boxes\ncigarette.butts\ncigarette.sheets\ncigarettes\nclay\nClay\nclothes\ncloves\ncoal\ncoat\nCoca.cola.bottle\ncoconut.oil\ncoffee\ncoin\ncollage\ncollagraph\ncolour\ncolour.and\ncolour.ans.sound..stereo.\ncoloured.graphite\ncoloured.light.bulbs\ncoloured.pencil\ncoloured.pencils\ncolumns\nCommercial.paint\ncompressor\ncompressors\ncomputer\nConcrete\nConté\nContractor.s.shed\nCooked.couscous\ncooker\ncopper\ncopper.sulphate\ncopperplate\ncopperplated\ncoral\ncord\nCork.panels\ncorrection.fluid\nCorrection.fluid\ncotton\nCotton\ncotton.costume\ncotton.thread\ncotton.wool\ncow\nCowhide\ncrayon\nCrayon\ncrude.oil\ncrystalline.particles\ncue.stand\ncues\ncurtains\nCut.paper\nDelabole.slate\ndesks\ndetritus\ndigging.tools\ndigital.image\ndigital.print\nDigital.print\nDigital.print.with.acrylic.paint\ndigital.prints\nDigital.prints.with.acrylic.paint\ndolls\ndoor\ndrawings\nDresden.D1.projector\ndress.shirt\ndry.pigment.paint\ndrypoint\nDrypoint\nduck.tape\ndye\nearth\nEarth\nearthenware\nEarthenware\nelectric.cable\nelectric.wire\nelectrical.a\nelectrical.cable\nelectrical.components\nelectrical.pump\nelectronic.circuit\nelephant.dung\nemulsion\nEmulsion\nenamel\nEnamel\nenamel.button\nenamel.paint\nEnamel.paint\nencre.de.Chine\nEngine\nengraving\nEngraving\nenvelopes\nepoxy\netching\nEtching\nEtching.with.hand.colouring\nexpanding.foam\neyebrow.pencil\neyeshadow\nfabric\nFabric\nfabric.patches\nfaced.particleboard\nfan.heater\nFeathers\nFed.2.type.camera\nfelt\nFelt\nfelt.pen\nFerric.oxide\nfibre.clay\nfibreboard\nFibreboard\nFibreboard.cabinet\nfibreboard.panels\nfibreboard.plinth\nfibreboard.with.clay\nfibreglass\nFibreglass\nfilm\nFilm\nFilm.16.mm\nflat.screen\nflat.screen.or.projector\nflax\nFlies\nFlint\nfloodlight\nflowers\nfluorescent.light\nfluorescent.lights\nfluorescent.site.jacket\nfluorescent.tubes\nflyers\nfoam\nfoam.core\nfoam.rubber\nfoil\nfoil.tape\nfood\nformaldehyde.solution\nformica\nFormica\nfound.posters\nframed.\nFuton.mattress\nGallery.lighting\ngalvanised.steel\ngalvanized.steel\ngel\ngelatin.silver.print\ngelatin.silver.printon.paper\ngelatine.silver.print.with.dye\nglas\nglass\nGlass\nglass.a\nglass.beads\nglass.box\nGlass.chandelier\nglass.vitrine.containing.beauty.and\nGlass.10.hardbacked.books\nglitter\nglove\nglue\ngobo\ngold.leaf\ngold.metallic.paint\nGold.paint\ngouache\nGouache\nGranite\ngraphite\nGraphite\ngrill\nGrpahite\nguitar.strings\ngun\nGunpowder\nhair\nhair.over.metal\nHand.coloured.photograph\nhand.colouring\nHand.cut.book\nhandcolouring\nhardboard\nhardboard.plinth\nhardboard.table\nhardwood\nhat\nheadphones\nHeart.to.Hear\nHeavy.goods.vehicle\nhelium.balloon\nhemp.cord\nhessian\nHessian\nhigh.definition\nhigh.definition_2\nhook\nhorn\nhosepipes\nHousehold.emulsion.paint\nhousehold.gloss.paint\nhousehold.paint\nHousehold.paint\nhuman.hair\nhydraulic.rams\nidentity.card\ninfrared.sensors\ningrained.dust\nink\nInk\ninteractive\nIris.print\niron\niron.powder\njacket\nJacket\njute\nkapok\nkey.\nKilkenny.limestone\nkitchen.utensils\nkite\nknife\nl\nlab.coat\nlacquer\nLacquered.wood\nladder\nlamb\nLambda.print\nlamps\nLamps\nlatex\nlead\nLead\nleather\nLeather\nleather.briefcase\nletterpress\nLetterpress\nlice\nlight\nlight.box\nlight.boxes\nlight.bulb\nlight.bulbs\nlight.control.unit\nlightbox\nLightboxes\nlighter\nlighting.system\nlights\nlinen\nlinocut\nLinocut\nLinocut.print\nlinoleum\nlithograph\nLithograph\nlithograph.with.plasticine\nLithograph.woodcut\nlithographs\nmagnetic.rods\nmagnets\nMahogany\nmango.seeds\nmannequin\nMannequin\nmap\nMap\nmap.pins\nmarble\nMarble\nmarble.base\nmarker.pen\nMarker.pen\nMDF\nMDF.backboard\nme\nmelamine\nmelanine.fibreboard\nMelinex\nmetal\nMetal\nMetal.bicycle\nmetal.bottle.caps\nmetal.bucket\nmetal.chain\nmetal.clamp\nmetal.clothes.rail\nMetal.cot\nmetal.detritus\nmetal.film.spool\nmetal.foil\nMetal.frame\nMetal.frames\nmetal.hook\nmetal.pipe\nmetal.sheet\nmetal.string\nMetal.table\nMetal.tin.cans\nmetal.wire\nmetalcut.prints\nmetallic.powder\nmezzotint\nMezzotint\nmica.flakes\nmicrophone\nmin.DV.cam.tape\nmini.disc.player\nmirror\nMirror\nmirrors\nMiss.Piggy.Bag..and.contents.\nMixed.media\nmobile.pho\nmodelling.putty\nmonitor\nmonitor.or.flat.screen\nmonitor.or.projection\nmono\nMonoprint\nmonotype\nMonotype\nmorse.code.unit\nmotor\nmotorised.base\nmoun\nMountain.bicycle\nMountain.peak.in.container\nmountaineering.equipment\nmounted\nmounted.onto.aluminium\nMud\nmultiple.projections\nmuslin\nMylar.screen\nnails\nnatural.fibres\nneon.gas\nneon.lights\nNeon.lights\nneon.paint\nnewspapers\nnotebook\nnylon\nnylon.straps\nnylon.strings\nNylon.tights\nnyloprints\noak\nOak\noak.table\noak.twig\noffice\noil\noil.paint\nOil.paint\noil.pastel\noil.stick\nOil.stick\noil.tint\non.47.panels\non.aluminium\non.aluminium.panel\non.board\non.cardboard\non.paper\non.paper.between.glass\non.paper.mounted\non.paper.mounted.onto.acrylic.glass\non.paper.mounted.onto.aluminium\non.paper.mounted.onto.aluminium.panel\non.paper.mounted.onto.aluminuim\non.paper.mounted.onto.board\non.paper.mounted.onto.panel\non.paper.mounted.onto.paper\non.paper.mounted.onto.Perspex\non.paper.mounted.onto.plastic\non.paper.with.chalk\non.paper.with.dry.transfert.print.mounted.onto.paper\non.paper.with.paint\non.papers\non.papers.with.paint\non.plastic\non.sticker.paper\non.vinyl.mounted.onto.aluminium\nopen.cell.foam\noptical.gelatin.silver.fibre.print\nor.video\nOrgan\nOstrich.egg\nother.m\nother.materia\nother.materials\nothers.materials\noxidised.brass\np\npaint\nPaint\nPaint.brush\npainted.aluminium\nPainted.aluminium\npainted.boards\nPainted.fibreboard\npainted.glass\npainted.MDF\nPainted.steel\npainted.wall\npainted.wall.text\npainted.wood\nPainted.wood\nPainted.wooden.building.with.asphalt.shingle.roof\npalm.fronds\nPalm.tree\npanel\npaper\nPaper\npaper.mounted\npaper.mounted.onto.aluminium\npaper.mounted.onto.aluminium.panel\npaper.mounted.onto.board\npaper.mounted.onto.foam.core\npaper.mounted.onto.muslin\npaper.mounted.onto.panel\npaper.mounted.onto.paper\npaper.pulp\npaper.tape\npaper.with.dry.transfer.print\npaper.with.dye\npaper.with.ink\npaper.with.oil.paint\npaper.with.watercolour\npaper..Verso..ink\npapers\nparaffin\nparaffin.lamp\npassport\npastel\nPastel\npatina\npeacock.feathers\npen\npendant.lamps\npeople\nPerformance\nperfume.flask.with.pouch\nperspex\nPerspex\nPewter\npharmaceutical.packaging\nphoto.etching\nPhoto.etching\nphotograph\nPhotograph\nphotograph.mounted\nPhotographic.contact.sheet\nphotographs\nPhotographs\nphotographs.\nPiano\npigment\nPigment\nPigment.transfer\npigments\nPine.plywood\npins\nplants\nplaster\nPlaster\nplaster.figures\nplaster.powder\nplasterboard\nplastic\nPlastic\nplastic.bag\nplastic.bags\nplastic.beads\nPlastic.beads\nplastic.box\nplastic.boxes\nplastic.cover\nPlastic.lids\nplastic.pearls\nplastic.pen.lids\nplastic.pillow\nplastic.shoe\nplastic.thread\nplastic.tubes\nplastic.watch.with.photograph\nplastic.watering.can\nplastic.watering.can..spray.bot\nplasticine\nPlasticine\nplatinium.print\nPlexiglas\nplexiglass\nplinth\nplug.boards\nplywood\nPlywood\nplywood.board\npo\nPolychromed.aluminium\nPolyester\npolyester.foam\npolyester.resin\nPolyester.resin\npolyester.textile\npolyfibre\npolymer\npolystyrene\npolystyrene.foam\npolythene\npolyurethane\npolyurethane.foam\nPolyurethane.resin\nPolyurethane.rubber\npolyvinyl.acetate.paint\nPolyvinyl.acetate.paint\nporcelain\nPorcelain\nporcelain.model\nporcelaine\nportable.keyboard.keys\nPortorino.marble\npostcard\nPostcard\npostcards\nposters\nPotato.print\nPowder.coated.aluminium\npowder.coated.steel\nPowder.coated.steel\npowder.paint\npowder.coated.steel_2\nPowder.coated.steel_2\nprint\nPrint\nprinted.map\nprinted.p\nprinted.paper\nPrinted.paper\nprinted.papers\nPrinted.papers\nprojection\nprojection.or.7.monitors\nprojection.or.monitor\npumps\nPVC\nradio\nradio.transmitter\nramin\nrecipes\nRecord.deck\nrecord.player\nrecorded.voice\nreel.to.reel.tape.deck\nrelief\nRelief\nRelief.print\nresin\nResin\nresin.base\nresin.block\nribbon\nRibbon\nRiver.mud\nRoman.vessel\nrope\nropes\nrosary.beads\nrubber\nRubber\nrubber.coated.steel\nRubber.inner.tubes\nrubber.padding\nsafety.pins\nsalt\nsand\nsatin\nsatin.ribbon\nsawdust\nSax.oil.paint\nscanachrome.print\nScanachrome.print\nscissors\nscreenprint\nScreenprint\nScreenprint.with.acrylic.varnish\nScreenprints\nscrews\nsea.shells\nseat\nsection.of.concrete.wall\nsellotape\nSellotape\nSequins\nsewing.machine\nsheep.excrement\nshellac\nshellac.resin\nshells\nShells\nshield\nshirt\nshoeboxes\nshoelace\nshoes\nShop.mannequin\nshown.as\nshown.as.video\nshredder\nsilicon.hose\nsilicon.tubing\nsilicone\nsilicone.adhesive\nsilicone.rubber\nSilicone.rubber\nsilk.tie\nsilkcreen.print\nsilkscreen\nSilkscreen\nSilkscreen.print\nsilver\nsilver.foil\nsilver.gelatin.print\nsilver.solder\nSilver.teapot\nSilverpoint\nsisal.rope\nsisal.string\nSlate\nslide\nSlide\nslides\nsmall.speakers\nsmoke\nsmoke.machine\nso\nsoap\nSoap\nsocks\nSofa\nsoftware\nSoftware\nSondor.playback.machine\nsound\nsound..mono\nsound..mono.\nsound..stereo.\nsound..surround.\nsound.recording\nsound..wood\nspeakers\nspices\nspinning.top\nspray.enamel\nspray.paint\nSpray.paint\nSprayed.Q.Cell\nsreenprint\nStack.of.printed.paper\nstainless.steel\nStainless.steel\nstainless.steel.ashtray\nstainless.steel.screws\nstainless.steel.tea.urn\nstainless.steel.teapot\nstainless.steel.wire\nstand\nstaple.gun\nStarfish\nsteel\nSteel\nsteel.bars\nSteel.bedsprings\nsteel.bracket\nsteel.brake.wires\nsteel.cable\nSteel.locker\nSteel.plate\nsteel.screws\nsteel.shelving\nsteel.wire\nstencil\nstereo\nstereo.\nstickers\nstockings\nstone.base\nstone..with.Jamie.Sargeant\nstool\nstoryboards\nstraw\nstrin\nstring\nstyrene\nstyrofoam\nsugar.paper\nsuitcase\nSuper.16.mm\nSuper.8\nSuper.8.mm\nsweet.wrappers\nswing.seat\nsynthetic.fibre\nsynthetic.fibres\ntable\nTable\ntables\ntape\ntar\ntarpaulin\ntaxidermy\ntea\ntelephone\nTempera\ntent\ntextile\nTextile\ntextiles\nTextiles\ntheatre.light\nthread\nthreads\ntickets\ntights\nTimber\ntissue\ntoilet\ntooth.b\ntorchlight\ntoy.eyeballs\ntracing.paper\ntracking.system\ntram.window\ntransfer.lettering\ntransparency\nTransparency\ntransparent.paper\ntree.branches\ntripod\ntrousers\ntwigs\ntwo.pack.car.lacquer\ntype\ntypescript\ntypewritten.ink\nUHF.Radio\numbrella\nunderpants\nvarnish\nvelvet\nvelvet.p\nVHS.tape\nvideo\nVideo\nvideo.camera\nviewfinder\nvinyl\nVinyl.dispersion\nvinyl.floor.covering\nvinyl.paint\nVinyl.record\nvinyl.seat\nVinyl.tape\nvinyl.text\nvinyl.wall.text\nVinyl.wall.text\nvinyl.wall.texts\nwalkman\nWalkman\nwall\nwall.clock\nWall.painting\nwall.text\nWallpaper\nWashers\nwater\nwatercolour\nWatercolour\nwax\nweb.search.program\nwebbing\nWestern.red.cedar\nwheels\nwhite\nwire\nwire.coat.hanger.\nwires\nwood\nWood\nWood.engraving\nwood.tray\nwood.varnish\nwood.veneer\nWoodblock\nwoodcut\nWoodcut\nWooden.abacus.beads\nwooden.base\nwooden.beams\nWooden.billiard.table\nWooden.birdhouse.with.metal.roof\nwooden.boxes\nWooden.cabinet\nwooden.ceiling.props\nwooden.chair\nWooden.chair\nWooden.construction\nWooden.desk\nwooden.dowels\nwooden.floor\nwooden.frames\nwooden.glass.negative.plate.carrier\nwooden.pallets\nwooden.panel\nwooden.planks\nwooden.platform\nwooden.steps\nwooden.table\nWooden.table\nwooden.trestle\nwool\nWool\nwork.overalls\nworks\nWorry.beads\nwraps\nzinc\nother\n\n\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#enhance", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#enhance", + "title": "5 - Iterative Search", + "section": "ENHANCE", + "text": "ENHANCE\n\nautoplot(lgbm_bayes_res, metric = \"mae\", type = \"performance\") +\n ylim(c(9, 14))" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#othering-2", - "href": "slides/advanced-05-feature-engineering-part-one.html#othering-2", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Othering", - "text": "Othering\nthreshold = 0.05 produces 11 columns.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\naquatint\ncanvas\ncolour\nEtching\ngelatin.silver.print\nLithograph\non.paper\npaper\nPhotograph\nScreenprint\nother\n\n\n\n\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n3\n\n\n0\n0\n0\n1\n0\n0\n0\n1\n0\n0\n0\n\n\n0\n0\n0\n1\n0\n0\n0\n1\n0\n0\n0\n\n\n0\n0\n0\n1\n0\n0\n0\n1\n0\n0\n0\n\n\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n1\n\n\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n1" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#your-turn-1", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#your-turn-1", + "title": "5 - Iterative Search", + "section": "Your turn", + "text": "Your turn\nLet’s try a different acquisition function: conf_bound(kappa).\nWe’ll use the objective argument to set it.\nChoose your own kappa value:\n\nLarger values will explore the space more.\n“Large” values are usually less than one.\n\nBonus points: Before the optimization is done, press <esc> and see what happens.\n\n\n\n−+\n10:00" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#your-turn-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#your-turn-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Your turn", - "text": "Your turn\n\nFigure out whether step_dummy_extract() or step_dummy_multi_choice() is most appropriate on the edge variable in leaf_data and apply it\n\n\n\n−+\n03:00" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#notes", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#notes", + "title": "5 - Iterative Search", + "section": "Notes", + "text": "Notes\n\nStopping tune_bayes() will return the current results.\nParallel processing can still be used to more efficiently measure each candidate point.\nThere are a lot of other iterative methods that you can use.\nThe finetune package also has functions for simulated annealing search." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#dimensionality-reduction-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#dimensionality-reduction-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Dimensionality Reduction", - "text": "Dimensionality Reduction\nWhat is it?\nTechniques that remove or alter features in order to have fewer but more informative features.\nWhy do we do it?\n\nredundant information\nineffective representation\ncomputational speed" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#finalizing-the-model", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#finalizing-the-model", + "title": "5 - Iterative Search", + "section": "Finalizing the Model", + "text": "Finalizing the Model\nLet’s say that we’ve tried a lot of different models and we like our lightgbm model the most.\nWhat do we do now?\n\nFinalize the workflow by choosing the values for the tuning parameters.\nFit the model on the entire training set.\nVerify performance using the test set.\nDocument and publish the model(?)" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#redundant-information", - "href": "slides/advanced-05-feature-engineering-part-one.html#redundant-information", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Redundant Information", - "text": "Redundant Information\nA feature with no information could be included.\n\nleaf_data |>\n count(outlying_contour)\n#> # A tibble: 1 × 2\n#> outlying_contour n\n#> <dbl> <int>\n#> 1 0 1907" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#locking-down-the-tuning-parameters", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#locking-down-the-tuning-parameters", + "title": "5 - Iterative Search", + "section": "Locking Down the Tuning Parameters", + "text": "Locking Down the Tuning Parameters\nWe can take the results of the Bayesian optimization and accept the best results:\n\nbest_param <- select_best(lgbm_bayes_res, metric = \"mae\")\nfinal_wflow <- \n lgbm_wflow %>% \n finalize_workflow(best_param)\nfinal_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: boost_tree()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> avg_price_per_room ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Boosted Tree Model Specification (regression)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> min_n = 2\n#> tree_depth = 11\n#> learn_rate = 0.0871075826616985\n#> loss_reduction = 0.00195652467829182\n#> stop_iter = 18\n#> \n#> Engine-Specific Arguments:\n#> num_threads = 1\n#> \n#> Computational engine: lightgbm" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#redundant-information-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#redundant-information-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Redundant Information", - "text": "Redundant Information\nOr duplicated or functionally equivalent features\n\nleaf_data |>\n ggplot(aes(area, equivalent_diameter)) +\n geom_point()\n\n\n\nFigure 1" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#the-final-fit", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#the-final-fit", + "title": "5 - Iterative Search", + "section": "The Final Fit", + "text": "The Final Fit\nWe can use individual functions:\nfinal_fit <- final_wflow %>% fit(data = hotel_train)\n\n# then predict() or augment() \n# then compute metrics\n\nRemember that there is also a convenience function to do all of this:\n\nset.seed(3893)\nfinal_res <- final_wflow %>% last_fit(hotel_split, metrics = reg_metrics)\nfinal_res\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [3749/1251]> train/test split <tibble [2 × 4]> <tibble [0 × 3]> <tibble [1,251 × 4]> <workflow>" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#ineffective-representation", - "href": "slides/advanced-05-feature-engineering-part-one.html#ineffective-representation", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Ineffective Representation", - "text": "Ineffective Representation\nWhile keeping our models in mind, we want to make sure the data is well-suited\n\nCorrelated data\n\nhard for some models\n\nlat/lon compared to distance/angle\n\nhard for most models" + "objectID": "archive/2024-08-posit-conf/advanced-05-iterative.html#test-set-results", + "href": "archive/2024-08-posit-conf/advanced-05-iterative.html#test-set-results", + "title": "5 - Iterative Search", + "section": "Test Set Results", + "text": "Test Set Results\n\n\n\nfinal_res %>% \n collect_predictions() %>% \n cal_plot_regression(\n truth = avg_price_per_room, \n estimate = .pred)\n\nTest set performance:\n\nfinal_res %>% collect_metrics()\n#> # A tibble: 2 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 9.60 Preprocessor1_Model1\n#> 2 rsq standard 0.949 Preprocessor1_Model1\n\n\n\n\n\n\n\n\n\n\n\n\nRecall that resampling predicted the MAE to be 9.601." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#ineffective-representation-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#ineffective-representation-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Ineffective Representation", - "text": "Ineffective Representation\n\n\nFigure 2" + "objectID": "archive/2024-08-posit-conf/annotations.html#section", + "href": "archive/2024-08-posit-conf/annotations.html#section", + "title": "Annotations", + "section": "👀", + "text": "👀\nThis page contains annotations for selected slides.\nThere’s a lot that we want to tell you. We don’t want people to have to frantically scribble down things that we say that are not on the slides.\nWe’ve added sections to this document with longer explanations and links to other resources." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#ineffective-representation-2", - "href": "slides/advanced-05-feature-engineering-part-one.html#ineffective-representation-2", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Ineffective Representation", - "text": "Ineffective Representation\n\n\nFigure 3" + "objectID": "archive/2024-08-posit-conf/annotations.html#data-splitting-and-spending", + "href": "archive/2024-08-posit-conf/annotations.html#data-splitting-and-spending", + "title": "Annotations", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nMore about the initial data split can be found in Chapter 3 of Applied Machine Learning for Tabular Data (AML4TD).\nIn particular, a three-way split into training, validation, and testing set can be done via\n\nset.seed(123)\ninitial_validation_split(forested, prop = c(0.6, 0.2))\n#> <Training/Validation/Testing/Total>\n#> <4264/1421/1422/7107>" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#computational-speed", - "href": "slides/advanced-05-feature-engineering-part-one.html#computational-speed", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Computational Speed", - "text": "Computational Speed\nDepending on what method we are using and how the data is affected by it, we could see a large reduction in features. This, in turn, leads to a smaller model that is faster to train on.\nOnly exploration and trial and error can determine whether you should use dimensionality reduction techniques. Knowing which methods do what helps you determine what to try." + "objectID": "archive/2024-08-posit-conf/annotations.html#what-is-set.seed", + "href": "archive/2024-08-posit-conf/annotations.html#what-is-set.seed", + "title": "Annotations", + "section": "What is set.seed()?", + "text": "What is set.seed()?\nWhat does set.seed() do?\nWe’ll use pseudo-random numbers (PRN) to partition the data into training and testing. PRN are numbers that emulate truly random numbers (but really are not truly random).\nThink of PRN as a box that takes a starting value (the “seed”) that produces random numbers using that starting value as an input into its process.\nIf we know a seed value, we can reproduce our “random” numbers. To use a different set of random numbers, choose a different seed value.\nFor example:\n\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\n# Get a new set of random numbers:\nset.seed(2)\nrunif(3)\n#> [1] 0.1848823 0.7023740 0.5733263\n\n# We can reproduce the old ones with the same seed\nset.seed(1)\nrunif(3)\n#> [1] 0.2655087 0.3721239 0.5728534\n\nIf we don’t set the seed, R uses the clock time and the process ID to create a seed. This isn’t reproducible.\nSince we want our code to be reproducible, we set the seeds before random numbers are used.\nIn theory, you can set the seed once at the start of a script. However, if we do interactive data analysis, we might unwittingly use random numbers while coding. In that case, the stream is not the same and we don’t get reproducible results.\nThe value of the seed is an integer and really has no meaning. Max has a script to generate random integers to use as seeds to “spread the randomness around”. It is basically:\n\ncat(paste0(\"set.seed(\", sample.int(10000, 5), \")\", collapse = \"\\n\"))\n#> set.seed(9725)\n#> set.seed(8462)\n#> set.seed(4050)\n#> set.seed(8789)\n#> set.seed(1301)" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#dimensionality-reduction-method", - "href": "slides/advanced-05-feature-engineering-part-one.html#dimensionality-reduction-method", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Dimensionality Reduction Method", - "text": "Dimensionality Reduction Method\n\nZero Variance removal\nPCA\n\nTruncated PCA\nSparse PCA\n\nNNMF\nUMAP\nIsomap" + "objectID": "archive/2024-08-posit-conf/annotations.html#what-is-wrong-with-this", + "href": "archive/2024-08-posit-conf/annotations.html#what-is-wrong-with-this", + "title": "Annotations", + "section": "What is wrong with this?", + "text": "What is wrong with this?\nIf we treat the preprocessing as a separate task, it raises the risk that we might accidentally overfit to the data at hand.\nFor example, someone might estimate something from the entire data set (such as the principle components) and treat that data as if it were known (and not estimated). Depending on the what was done with the data, consequences in doing that could be:\n\nYour performance metrics are slightly-to-moderately optimistic (e.g. you might think your accuracy is 85% when it is actually 75%)\nA consequential component of the analysis is not right and the model just doesn’t work.\n\nThe big issue here is that you won’t be able to figure this out until you get a new piece of data, such as the test set.\nA really good example of this is in ‘Selection bias in gene extraction on the basis of microarray gene-expression data’. The authors re-analyze a previous publication and show that the original researchers did not include feature selection in the workflow. Because of that, their performance statistics were extremely optimistic. In one case, they could do the original analysis on complete noise and still achieve zero errors.\nGenerally speaking, this problem is referred to as data leakage. Some other references:\n\nOverfitting to Predictors and External Validation\nAre We Learning Yet? A Meta Review of Evaluation Failures Across Machine Learning\nNavigating the pitfalls of applying machine learning in genomics\nA review of feature selection techniques in bioinformatics\nOn Over-fitting in Model Selection and Subsequent Selection Bias in Performance Evaluation" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#restrictions", - "href": "slides/advanced-05-feature-engineering-part-one.html#restrictions", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Restrictions", - "text": "Restrictions\nAll the methods shown today will not be able to handle\n\nmissing data\nNon-numeric data" + "objectID": "archive/2024-08-posit-conf/annotations.html#brier-score", + "href": "archive/2024-08-posit-conf/annotations.html#brier-score", + "title": "Annotations", + "section": "Brier score", + "text": "Brier score\nThe Brier score measures how close a model probability estimate is to its best possible value (i.e., zero or one).\nIn the best case, the model is perfect, and every prediction equals 0.0 or 1.0 (depending on the true class). In this case, the Brier score is zero.\nWhen the model is uninformative and there are two classes, the worst-case values range from 0.25 to about 0.50. Imagine that the model predicts the same noninformative prediction of 50% (basically “¯\\(ツ)/¯”). In that case, every prediction is either \\((0.00 - 0.50)^2\\) or \\((1.00 - 0.50)^2\\). The average of those is 0.25.\nThere are many different ways a model can be bad though, and some of these will produce Brier scores between 0.25 and 0.50." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#why-not-t-sne", - "href": "slides/advanced-05-feature-engineering-part-one.html#why-not-t-sne", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Why not t-SNE?", - "text": "Why not t-SNE?\nOne of the main requirements for a feature engineering method is that you can reapply the trained transformation done on the training data set to the testing data set.\nThis is not possible with t-SNE as it is an iterative method that shifts observations in the lower-dimensional space based on their distances to points in the higher-dimensional space.\nIt doesn’t create a mapping that can be reused.\n\nVisualizing Data using t-SNE, Laurens van der Maaten and Geoffrey Hinton, Journal of Machine Learning Research, 2008." + "objectID": "archive/2024-08-posit-conf/annotations.html#dangers-of-overfitting", + "href": "archive/2024-08-posit-conf/annotations.html#dangers-of-overfitting", + "title": "Annotations", + "section": "Dangers of overfitting", + "text": "Dangers of overfitting\nSee the “Overfitting” chapter of AML4TD for more information." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#pca", - "href": "slides/advanced-05-feature-engineering-part-one.html#pca", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "PCA", - "text": "PCA\nPrincipal Compoment Analysis is a linear combination of the original data such that most of the variation is captured in the first variable, then the second, then the third, and so on.\n\nFEAZ, FES" + "objectID": "archive/2024-08-posit-conf/annotations.html#where-are-the-fitted-models", + "href": "archive/2024-08-posit-conf/annotations.html#where-are-the-fitted-models", + "title": "Annotations", + "section": "Where are the fitted models?", + "text": "Where are the fitted models?\nThe primary purpose of resampling is to estimate model performance. The models are almost never needed again.\nAlso, if the data set is large, the model object may require a lot of memory to save so, by default, we don’t keep them.\nFor more advanced use cases, you can extract and save them. See:\n\nhttps://www.tmwr.org/resampling.html#extract\nhttps://www.tidymodels.org/learn/models/coefficients/ (an example)" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#pca-algorithm", - "href": "slides/advanced-05-feature-engineering-part-one.html#pca-algorithm", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "PCA Algorithm", - "text": "PCA Algorithm\nThe first principal component of a set of features \\(X_1, X_2, ..., X_p\\) is the normalized linear combination of the features.\n\\[\nZ_1 = \\phi_{11} X_1 + \\phi_{21} X_2 + ... + \\phi_{p1}X_p\n\\]\nthat has the largest variance under the constraint that \\(\\sum_{j=1}^p \\phi_{j1}^2 = 1\\).\nwe refer to \\(\\phi_{11}, ..., \\phi_{p1}\\) as the loadings of the first principal component.\nAnd think of them as the loading vector \\(\\phi_1\\)." + "objectID": "archive/2024-08-posit-conf/annotations.html#resampling-strategy", + "href": "archive/2024-08-posit-conf/annotations.html#resampling-strategy", + "title": "Annotations", + "section": "Resampling Strategy", + "text": "Resampling Strategy\nThese data have a time component, and while not a typical time series data set, we have the option to use a time series resampling method.\nAn example is shown in the extra slides “Case Study on Transportation”.\nConsider the agent data. Cross-validation may not group all of an agent’s data into the analysis or assessment sets. In this case, our analysis data might have future data that is later than the agent’s data in the assessment set." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#pca-algorithm-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#pca-algorithm-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "PCA Algorithm", - "text": "PCA Algorithm\nsince we have \\(z_{i1} = \\phi_{11} x_{i1} + \\phi_{21} x_{i2} + ... + \\phi_{p1}x_{ip}\\), then we can write\n\\[\n\\underset{\\phi_{11}, ..., \\phi_{p1}}{\\text{maximize}} \\left\\{ \\dfrac{1}{n} \\sum^n_{i=j} z_{i1} ^2 \\right\\} \\quad \\text{subject to} \\quad \\sum_{j=1}^p \\phi_{j1}^2 = 1\n\\]\nWe are, in essence, maximizing the sample variance of the \\(n\\) values of \\(z_{i1}\\).\nWe refer to \\(z_{11}, ..., z_{n1}\\) as the scores of the first principal component." + "objectID": "archive/2024-08-posit-conf/annotations.html#different-types-of-grids", + "href": "archive/2024-08-posit-conf/annotations.html#different-types-of-grids", + "title": "Annotations", + "section": "Different types of grids", + "text": "Different types of grids\nMore on space-filling designs in Chapters 4 and 5 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences.\nIn the next version of tune (version 1.3.0) an improved set of space-filling designs will be the first choice if you ask for an automated grid." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#pca-algorithm-2", - "href": "slides/advanced-05-feature-engineering-part-one.html#pca-algorithm-2", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "PCA Algorithm", - "text": "PCA Algorithm\nLuckily, this can be solved using techniques from Linear Algebra, more specifically, it can be solved using an eigen decomposition.\nOne of the main strengths of PCA is that you don’t need to use optimization to get the results without approximations." + "objectID": "archive/2024-08-posit-conf/annotations.html#update-parameter-ranges", + "href": "archive/2024-08-posit-conf/annotations.html#update-parameter-ranges", + "title": "Annotations", + "section": "Update parameter ranges", + "text": "Update parameter ranges\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object %>% \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) %>% \n step_spline_natural(dis, deg_free = tune()) %>% \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#pca-algorithm-3", - "href": "slides/advanced-05-feature-engineering-part-one.html#pca-algorithm-3", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "PCA Algorithm", - "text": "PCA Algorithm\nOnce the first principal component is calculated, we can calculate the second principal component.\nWe find the second principal component \\(Z_2\\) as a linear combination of \\(X_1, ..., X_p\\) that has the maximal variance out of the linear combinations that are uncorrelated with \\(Z_1\\)\nthis is the same as saying that \\(\\phi_2\\) should be orthogonal to the direction \\(\\phi_1\\)" + "objectID": "archive/2024-08-posit-conf/annotations.html#early-stopping-for-boosted-trees", + "href": "archive/2024-08-posit-conf/annotations.html#early-stopping-for-boosted-trees", + "title": "Annotations", + "section": "Early stopping for boosted trees", + "text": "Early stopping for boosted trees\nWhen deciding on the number of boosting iterations, there are two main strategies:\n\nDirectly tune it (trees = tune())\nSet it to one value and tune the number of early stopping iterations (trees = 500, stop_iter = tune()).\n\nEarly stopping is when we monitor the performance of the model. If the model doesn’t make any improvements for stop_iter iterations, training stops.\nHere’s an example where, after eleven iterations, performance starts to get worse.\n\n\n\n\n\n\n\n\n\nThis is likely due to over-fitting so we stop the model at eleven boosting iterations.\nEarly stopping usually has good results and takes far less time.\nWe could an engine argument called validation here. That’s not an argument to any function in the lightgbm package.\nbonsai has its own wrapper around (lightgbm::lgb.train()) called bonsai::train_lightgbm(). We use that here and it has a validation argument.\nHow would you know that? There are a few different ways:\n\nLook at the documentation in ?boost_tree and click on the lightgbm entry in the engine list.\nCheck out the pkgdown reference website https://parsnip.tidymodels.org/reference/index.html\nRun the translate() function on the parsnip specification object.\n\nThe first two options are best since they tell you a lot more about the particularities of each model engine (there are a lot for lightgbm)." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#how-is-that-a-dimensionality-reduction-method", - "href": "slides/advanced-05-feature-engineering-part-one.html#how-is-that-a-dimensionality-reduction-method", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "How is that a dimensionality reduction method?", - "text": "How is that a dimensionality reduction method?\nBy itself, it isn’t, as it rotates all the features in the feature space.\nIt becomes a dimensionality reduction method if we only calculate some of the principal components.\nThis is typically done by retaining a specific number of components or as a threshold on the variance explained." + "objectID": "archive/2024-08-posit-conf/annotations.html#gaussian-processes-and-optimization", + "href": "archive/2024-08-posit-conf/annotations.html#gaussian-processes-and-optimization", + "title": "Annotations", + "section": "Gaussian Processes and Optimization", + "text": "Gaussian Processes and Optimization\nSome other references for GP’s:\n\nChapter 5 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences\nBayesian Optimization, Chapter 3 (pdf)\nGaussian Processes for Machine Learning (pdf)" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#different-percent-variance-plots", - "href": "slides/advanced-05-feature-engineering-part-one.html#different-percent-variance-plots", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "4 different percent variance plots", - "text": "4 different percent variance plots" + "objectID": "archive/2024-08-posit-conf/annotations.html#acquisition-functions", + "href": "archive/2024-08-posit-conf/annotations.html#acquisition-functions", + "title": "Annotations", + "section": "Acquisition Functions", + "text": "Acquisition Functions\nMore references:\n\nChapter 7 of Surrogates: Gaussian process modeling, design, and optimization for the applied sciences\nBayesian Optimization, Chapter 6 (pdf)\nGaussian Processes for Machine Learning" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#applying-pca-with-recipes", - "href": "slides/advanced-05-feature-engineering-part-one.html#applying-pca-with-recipes", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Applying PCA with recipes ", - "text": "Applying PCA with recipes \nEither use the num_comp argument.\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca(all_numeric_predictors(), num_comp = 5)\n\n\nor using the threshold argument\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca(all_numeric_predictors(), threshold = 0.8)" + "objectID": "archive/2024-08-posit-conf/annotations.html#per-agent-statistics", + "href": "archive/2024-08-posit-conf/annotations.html#per-agent-statistics", + "title": "Annotations", + "section": "Per-agent statistics", + "text": "Per-agent statistics\nThe effect encoding method essentially takes the effect of a variable, like agent, and makes a data column for that effect. In our example, affect of the agent on the ADR is quantified by a model and then added as a data column to be used in the model.\nSuppose agent Max has a single reservation in the data and it had an ADR of €200. If we used a naive estimate for Max’s effect, the model is being told that Max should always produce an effect of €200. That’s a very poor estimate since it is from a single data point.\nContrast this with seasoned agent Davis, who has taken 250 reservations with an average ADR of €100. Davis’s mean is more predictive because it is estimated with better data (i.e., more total reservations). Partial pooling leverages the entire data set and can borrow strength from all of the agents. It is a common tool in Bayesian estimation and non-Bayesian mixed models. If a agent’s data is of good quality, the partial pooling effect estimate is closer to the raw mean. Max’s data is not great and is “shrunk” towards the center of the overall average. Since there is so little known about Max’s reservation history, this is a better effect estimate (until more data is available for him).\nThe Stan documentation has a pretty good vignette on this: https://cran.r-project.org/web/packages/rstanarm/vignettes/pooling.html\nAlso, Bayes Rules! has a nice section on this: https://www.bayesrulesbook.com/chapter-15.html\nSince this example has a numeric outcome, partial pooling is very similar to the James–Stein estimator: https://en.wikipedia.org/wiki/James–Stein_estimator" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#your-turn-2", - "href": "slides/advanced-05-feature-engineering-part-one.html#your-turn-2", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Your turn", - "text": "Your turn\n\nApply PCA using step_pca() to leaf_data data set.\nExperiment with different values of num_comp and or threshold.\n\n\n\n−+\n05:00" + "objectID": "archive/2024-08-posit-conf/annotations.html#agent-effects", + "href": "archive/2024-08-posit-conf/annotations.html#agent-effects", + "title": "Annotations", + "section": "Agent effects", + "text": "Agent effects\nEffect encoding might result in a somewhat circular argument: the column is more likely to be important to the model since it is the output of a separate model. The risk here is that we might over-fit the effect to the data. For this reason, it is super important to make sure that we verify that we aren’t overfitting by checking with resampling (or a validation set).\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage though." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#pca-pros-and-cons", - "href": "slides/advanced-05-feature-engineering-part-one.html#pca-pros-and-cons", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "PCA Pros and Cons", - "text": "PCA Pros and Cons\n\n\nPros\n\nFast\nReliable\nExact results (up to sign changes)\n\n\nCons\n\nComputational time is linear in the number of columns\nCan be quite hard to interpret" + "objectID": "archive/2024-08-posit-conf/annotations.html#a-recipe---handle-correlations", + "href": "archive/2024-08-posit-conf/annotations.html#a-recipe---handle-correlations", + "title": "Annotations", + "section": "A recipe - handle correlations", + "text": "A recipe - handle correlations\nIn this code chunk, What’s the story with !!stations?\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nstations is a vector of names of 20 columns that we want to use in the steps. If the list were shorter, we could type them in (e.g., c(\"col1\", \"col2\") etc.).\nThe vector lives in our global workspace, and if we are in parallel, the worker processes might not have access to stations. The !! (frequently said as “bang bang”) inserts the actual contents of the vector into the all_of() calls so that it looks like you just typed it in.\nThis means that the parallel process workers have a copy of the data in their reach and the code will run without error." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#truncated-pca", - "href": "slides/advanced-05-feature-engineering-part-one.html#truncated-pca", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Truncated PCA", - "text": "Truncated PCA\n\nComputational time is linear in the number of columns\n\nBy default, step_pca() calculates all the loading vectors. And then subset them down to what you need.\nInstead, we can use a different implementation that only calculates what you need. This is what we call truncated PCA.\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#chicago-l-train-data", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#chicago-l-train-data", + "title": "Case Study on Transportation", + "section": "Chicago L-Train data", + "text": "Chicago L-Train data\nSeveral years worth of pre-pandemic data were assembled to try to predict the daily number of people entering the Clark and Lake elevated (“L”) train station in Chicago.\nMore information:\n\nSeveral Chapters in Feature Engineering and Selection.\n\nStart with Section 4.1\nSee Section 1.3\n\nVideo: The Global Pandemic Ruined My Favorite Data Set" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#truncated-pca-with-recipes", - "href": "slides/advanced-05-feature-engineering-part-one.html#truncated-pca-with-recipes", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Truncated PCA with recipes ", - "text": "Truncated PCA with recipes \nCan only be done using num_comp\n\nlibrary(embed)\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_truncated(all_numeric_predictors(), num_comp = 5)" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#predictors", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#predictors", + "title": "Case Study on Transportation", + "section": "Predictors", + "text": "Predictors\n\nthe 14-day lagged ridership at this and other stations (units: thousands of rides/day)\nweather data\nhome/away game schedules for Chicago teams\nthe date\n\nThe data are in modeldata. See ?Chicago." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#sparse-pca", - "href": "slides/advanced-05-feature-engineering-part-one.html#sparse-pca", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Sparse PCA", - "text": "Sparse PCA\n\nCan be quite hard to interpret\n\nEvery component is a linear combination of all predictors.\n\\[\n\\begin{alignat}{4}\nPC1 &= cyl \\cdot -0.021 &&+ disp \\cdot -0.85 &&+ hp \\cdot -0.52\\\\\nPC2 &= cyl \\cdot 0.013 &&+ disp \\cdot -0.52 &&+ hp \\cdot 0.85\\\\\nPC3 &= cyl \\cdot -0.12 &&+ disp \\cdot 0.016 &&+ hp \\cdot 0.081\\\\\nPC4 &= cyl \\cdot -0.22 &&+ disp \\cdot -0.0061 &&+ hp \\cdot 0.033\\\\\nPC5 &= cyl \\cdot 0.73 &&+ disp \\cdot -0.014 &&+ hp \\cdot 0.0016\n\\end{alignat}\n\\]\nIf we could force some of the loadings to be 0, it would reduce things a lot.\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#l-train-locations", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#l-train-locations", + "title": "Case Study on Transportation", + "section": "L Train Locations", + "text": "L Train Locations" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#sparse-pca-with-recipes", - "href": "slides/advanced-05-feature-engineering-part-one.html#sparse-pca-with-recipes", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Sparse PCA with recipes ", - "text": "Sparse PCA with recipes \nCan only be done using num_comp.\nThe predictor_prop argument is used to determine how many zeroes in the loadings.\n\nlibrary(embed)\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_sparse(all_numeric_predictors(), num_comp = 5, predictor_prop = 0.8)" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#your-turn-explore-the-data", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#your-turn-explore-the-data", + "title": "Case Study on Transportation", + "section": "Your turn: Explore the Data", + "text": "Your turn: Explore the Data\nTake a look at these data for a few minutes and see if you can find any interesting characteristics in the predictors or the outcome.\n\nlibrary(tidymodels)\nlibrary(rules)\ndata(\"Chicago\")\ndim(Chicago)\n#> [1] 5698 50\nstations\n#> [1] \"Austin\" \"Quincy_Wells\" \"Belmont\" \"Archer_35th\" \n#> [5] \"Oak_Park\" \"Western\" \"Clark_Lake\" \"Clinton\" \n#> [9] \"Merchandise_Mart\" \"Irving_Park\" \"Washington_Wells\" \"Harlem\" \n#> [13] \"Monroe\" \"Polk\" \"Ashland\" \"Kedzie\" \n#> [17] \"Addison\" \"Jefferson_Park\" \"Montrose\" \"California\"\n\n\n\n\n−+\n05:00" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#sparse-pca-with-recipes-1", - "href": "slides/advanced-05-feature-engineering-part-one.html#sparse-pca-with-recipes-1", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Sparse PCA with recipes ", - "text": "Sparse PCA with recipes \npredictor_prop = 0.8\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_sparse(all_numeric_predictors(), num_comp = 4, predictor_prop = 0.8) |>\n prep() |>\n tidy(number = 2) |>\n pivot_wider(names_from = component, values_from = value) |>\n select(-id)\n#> # A tibble: 10 × 5\n#> terms PC1 PC2 PC3 PC4\n#> <chr> <dbl> <dbl> <dbl> <dbl>\n#> 1 cyl -0.503 0 0.0755 0 \n#> 2 disp -0.504 0 0 0.227 \n#> 3 hp -0.377 -0.228 -0.0985 0 \n#> 4 drat 0.267 -0.262 0 0.911 \n#> 5 wt -0.420 0.105 -0.383 0.236 \n#> 6 qsec 0 0.473 -0.425 0.0966 \n#> 7 vs 0.313 0.203 -0.429 -0.0774 \n#> 8 am 0.0687 -0.440 0.191 -0.00802\n#> 9 gear 0 -0.482 -0.237 -0.208 \n#> 10 carb -0.00616 -0.422 -0.617 -0.0654" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#splitting-with-chicago-data", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#splitting-with-chicago-data", + "title": "Case Study on Transportation", + "section": "Splitting with Chicago data ", + "text": "Splitting with Chicago data \nLet’s put the last two weeks of data into the test set. initial_time_split() can be used for this purpose:\n\ndata(Chicago)\n\nchi_split <- initial_time_split(Chicago, prop = 1 - (14/nrow(Chicago)))\nchi_split\n#> <Training/Testing/Total>\n#> <5684/14/5698>\n\nchi_train <- training(chi_split)\nchi_test <- testing(chi_split)\n\n## training\nnrow(chi_train)\n#> [1] 5684\n \n## testing\nnrow(chi_test)\n#> [1] 14" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#sparse-pca-with-recipes-2", - "href": "slides/advanced-05-feature-engineering-part-one.html#sparse-pca-with-recipes-2", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Sparse PCA with recipes ", - "text": "Sparse PCA with recipes \npredictor_prop = 0.2\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_sparse(all_numeric_predictors(), num_comp = 4, predictor_prop = 0.2) |>\n prep() |>\n tidy(number = 2) |>\n pivot_wider(names_from = component, values_from = value) |>\n select(-id)\n#> # A tibble: 10 × 5\n#> terms PC1 PC2 PC3 PC4\n#> <chr> <dbl> <dbl> <dbl> <dbl>\n#> 1 cyl 0.712 0 0 0 \n#> 2 disp 0.702 0 0 0 \n#> 3 hp 0 0 0 0 \n#> 4 drat 0 0 0 0.978\n#> 5 wt 0 0 -0.278 0 \n#> 6 qsec 0 0.666 0 0 \n#> 7 vs 0 0 0 0 \n#> 8 am 0 0 0 0.211\n#> 9 gear 0 -0.746 0 0 \n#> 10 carb 0 0 -0.961 0" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#time-series-resampling", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#time-series-resampling", + "title": "Case Study on Transportation", + "section": "Time series resampling", + "text": "Time series resampling\nOur Chicago data is over time. Regular cross-validation, which uses random sampling, may not be the best idea.\nWe can emulate our training/test split by making similar resamples.\n\nFold 1: Take the first X years of data as the analysis set, the next 2 weeks as the assessment set.\nFold 2: Take the first X years + 2 weeks of data as the analysis set, the next 2 weeks as the assessment set.\nand so on" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#nnmf", - "href": "slides/advanced-05-feature-engineering-part-one.html#nnmf", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "NNMF", - "text": "NNMF\nNon-Negative Matrix Factorization is conceptually similar to PCA, but it has different objectives.\nPCA aims to generate uncorrelated components that maximize the variances. One component at a time.\nNNMF, on the other hand, simultaneously optimizes all the components under the constraint that all the loadings are non-negative. While the data is also non-negative.\n\nFEAZ FES" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#rolling-forecast-origin-resampling", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#rolling-forecast-origin-resampling", + "title": "Case Study on Transportation", + "section": "Rolling forecast origin resampling", + "text": "Rolling forecast origin resampling\n\n\nThis image shows overlapping assessment sets. We will use non-overlapping data but it could be done wither way." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#nnmf-restriction", - "href": "slides/advanced-05-feature-engineering-part-one.html#nnmf-restriction", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "NNMF restriction", - "text": "NNMF restriction\nThe data has to be non-negative, i.e., 0 or higher.\nThis might feel like a pretty big restriction. And that is not wrong. But a lot of data sets end up being naturally non-negative.\nOr could at least be turned into non-negative ones with transformations. As long as you don’t scale them below 0.\nIt makes for much easier interpretations as the loadings don’t cancel each other out like they do for PCA." + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n\n\n\n\n )\n\nUse the date column to find the date data." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#nnmf-pros-and-cons", - "href": "slides/advanced-05-feature-engineering-part-one.html#nnmf-pros-and-cons", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "NNMF Pros and Cons", - "text": "NNMF Pros and Cons\n\n\n\nMore interpretable results\nPulls out better structures\n\n\n\nData must be non-negative\nComputationally expensive\nTraining depends on the seed" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-1", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-1", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n\n\n\n )\n\nOur units will be weeks." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#nnmf-with-recipes", - "href": "slides/advanced-05-feature-engineering-part-one.html#nnmf-with-recipes", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "NNMF with recipes ", - "text": "NNMF with recipes \n\nset.seed(1234)\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_nnmf_sparse(all_numeric_predictors(), num_comp = 2) |>\n prep() |>\n tidy(number = 1) |>\n pivot_wider(names_from = component, values_from = value) |>\n select(-id)\n#> # A tibble: 10 × 3\n#> terms NNMF1 NNMF2\n#> <chr> <dbl> <dbl>\n#> 1 am 0 0.00631\n#> 2 carb 0.00428 0.0249 \n#> 3 cyl 0.0134 0.0177 \n#> 4 disp 0.639 0 \n#> 5 drat 0.00545 0.0188 \n#> 6 gear 0.00509 0.0240 \n#> 7 hp 0.294 0.840 \n#> 8 qsec 0.0316 0.0606 \n#> 9 vs 0.000293 0.00247\n#> 10 wt 0.00742 0.00519" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-2", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-2", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15 \n \n \n )\n\nEvery analysis set has 15 years of data" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#umap", - "href": "slides/advanced-05-feature-engineering-part-one.html#umap", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "UMAP", - "text": "UMAP\nUniform Manifold Approximation and Projection is another method that takes high-dimensional data and transforms it into a lower-dimensional space.\nRuns relatively fast and is popular in visualizations." + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-3", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-3", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n\n )\n\nEvery assessment set has 2 weeks of data" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#umap-algorithm", - "href": "slides/advanced-05-feature-engineering-part-one.html#umap-algorithm", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "UMAP Algorithm", - "text": "UMAP Algorithm\nRough algorithm\n\nUse spectral embedding to embed points in a low-dimensional space\nCalculate similarity scores between points based on the original data set\nRandomly samples a pair of points based on their similarity scores\nFlips a coin to decide which of the pair of points to give to the other one\nRandomly picks a non-neighbor point to move away from\nMoves the selected point towards its neighbor and away from its non-neighbor\nRepeat 3-6" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-4", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#times-series-resampling-4", + "title": "Case Study on Transportation", + "section": "Times series resampling ", + "text": "Times series resampling \n\nchi_rs <-\n chi_train %>%\n sliding_period(\n index = \"date\", \n period = \"week\",\n lookback = 52 * 15,\n assess_stop = 2,\n step = 2 \n )\n\nIncrement by 2 weeks so that there are no overlapping assessment sets.\n\nchi_rs$splits[[1]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-07\" \"2016-01-20\"\nchi_rs$splits[[2]] %>% assessment() %>% pluck(\"date\") %>% range()\n#> [1] \"2016-01-21\" \"2016-02-03\"" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#umap-parameters", - "href": "slides/advanced-05-feature-engineering-part-one.html#umap-parameters", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "UMAP parameters", - "text": "UMAP parameters\n\nn_neighbors\n\nDetermines how many points are considered neighbors. A point counts as its own neighbor. Lower values lead to a local view.\n\nmin_dist\n\nDetermines how close points are allowed to be to each other in the low-dimensional space.\n\nmetric\n\nHow distances are calculated in the input data: euclidean, manhattan, jaccard, etc." + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#our-resampling-object", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#our-resampling-object", + "title": "Case Study on Transportation", + "section": "Our resampling object ", + "text": "Our resampling object \n\n\n\nchi_rs\n#> # Sliding period resampling \n#> # A tibble: 16 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5463/14]> Slice01\n#> 2 <split [5467/14]> Slice02\n#> 3 <split [5467/14]> Slice03\n#> 4 <split [5467/14]> Slice04\n#> 5 <split [5467/14]> Slice05\n#> 6 <split [5467/14]> Slice06\n#> 7 <split [5467/14]> Slice07\n#> 8 <split [5467/14]> Slice08\n#> 9 <split [5467/14]> Slice09\n#> 10 <split [5467/14]> Slice10\n#> 11 <split [5467/14]> Slice11\n#> 12 <split [5467/14]> Slice12\n#> 13 <split [5467/14]> Slice13\n#> 14 <split [5467/14]> Slice14\n#> 15 <split [5467/14]> Slice15\n#> 16 <split [5467/11]> Slice16\n\n\n\n\nWe will fit 16 models on 16 slightly different analysis sets.\nEach will produce a separate performance metrics.\nWe will average the 16 metrics to get the resampling estimate of that statistic." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#umap-hesitancy", - "href": "slides/advanced-05-feature-engineering-part-one.html#umap-hesitancy", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "UMAP hesitancy", - "text": "UMAP hesitancy\nDue to the flexibility of how this method works, it is almost always possible to generate graphs that appear to have insights in them." + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#feature-engineering-with-recipes", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#feature-engineering-with-recipes", + "title": "Case Study on Transportation", + "section": "Feature engineering with recipes ", + "text": "Feature engineering with recipes \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train)\n\nBased on the formula, the function assigns columns to roles of “outcome” or “predictor”" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#which-was-created-with-random-data", - "href": "slides/advanced-05-feature-engineering-part-one.html#which-was-created-with-random-data", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Which was created with random data?", - "text": "Which was created with random data?\n\n\nFigure 4" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe", + "title": "Case Study on Transportation", + "section": "A recipe", + "text": "A recipe\n\nsummary(chi_rec)\n#> # A tibble: 50 × 4\n#> variable type role source \n#> <chr> <list> <chr> <chr> \n#> 1 Austin <chr [2]> predictor original\n#> 2 Quincy_Wells <chr [2]> predictor original\n#> 3 Belmont <chr [2]> predictor original\n#> 4 Archer_35th <chr [2]> predictor original\n#> 5 Oak_Park <chr [2]> predictor original\n#> 6 Western <chr [2]> predictor original\n#> 7 Clark_Lake <chr [2]> predictor original\n#> 8 Clinton <chr [2]> predictor original\n#> 9 Merchandise_Mart <chr [2]> predictor original\n#> 10 Irving_Park <chr [2]> predictor original\n#> # ℹ 40 more rows" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#all-of-them-different-n_neighbors", - "href": "slides/advanced-05-feature-engineering-part-one.html#all-of-them-different-n_neighbors", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "All of them! Different n_neighbors", - "text": "All of them! Different n_neighbors\n\n\nFigure 5" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) \n\nThis creates three new columns in the data based on the date. Note that the day-of-the-week column is a factor." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#umap-with-recipes", - "href": "slides/advanced-05-feature-engineering-part-one.html#umap-with-recipes", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "umap with recipes ", - "text": "umap with recipes \n\nlibrary(embed)\nset.seed(1234)\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_umap(all_numeric_predictors()) |>\n prep() |>\n bake(NULL)\n#> # A tibble: 32 × 3\n#> mpg UMAP1 UMAP2\n#> <dbl> <dbl> <dbl>\n#> 1 21 1.50 2.26\n#> 2 21 1.26 2.02\n#> 3 22.8 2.17 4.19\n#> 4 21.4 -1.38 -1.67\n#> 5 18.7 -2.67 -2.66\n#> 6 18.1 -0.759 -1.57\n#> 7 14.3 -3.19 -3.37\n#> 8 24.4 1.31 3.91\n#> 9 22.8 2.06 2.46\n#> 10 19.2 0.876 2.31\n#> # ℹ 22 more rows" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-1", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-1", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) \n\nAdd indicators for major holidays. Specific holidays, especially those non-USA, can also be generated.\nAt this point, we don’t need date anymore. Instead of deleting it (there is a step for that) we will change its role to be an identification variable.\n\nWe might want to change the role (instead of removing the column) because it will stay in the data set (even when resampled) and might be useful for diagnosing issues." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#isomap", - "href": "slides/advanced-05-feature-engineering-part-one.html#isomap", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Isomap", - "text": "Isomap\nIsometric mapping is a non-linear dimensionality reduction method.\nThis is another method that uses distances between points to produce graphs of neighboring points. Where this method is different than other methods is that it uses geodesic distances as opposed to straight-line distances.\nThe geodesic distance is the sum of edge weights along the shortest path between two points.\nThe eigenvectors of the deodesic distance metric are then used to represent the new coordinates." + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-2", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---work-with-dates-2", + "title": "Case Study on Transportation", + "section": "A recipe - work with dates ", + "text": "A recipe - work with dates \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE)\n\ndate is still in the data set but tidymodels knows not to treat it as an analysis column.\nupdate_role_requirements() is needed to make sure that this column is required when making new data points. The help page has a good discussion about the nuances." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#isomap-algorithm", - "href": "slides/advanced-05-feature-engineering-part-one.html#isomap-algorithm", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Isomap Algorithm", - "text": "Isomap Algorithm\nA very high-level description of the Isomap algorithm is given below.\n\nFind the neighbors for each point\nConstruct the neighborhood graph, using Euclidean distance as edge length\nCalculate the shortest path between each pair of points\nUse Multidimensional scaling to compute a lower-dimensional embedding" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---remove-constant-columns", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---remove-constant-columns", + "title": "Case Study on Transportation", + "section": "A recipe - remove constant columns ", + "text": "A recipe - remove constant columns \n\nchi_rec <- \n recipe(ridership ~ ., data = chi_train) %>% \n step_date(date, features = c(\"dow\", \"month\", \"year\")) %>% \n step_holiday(date) %>% \n update_role(date, new_role = \"id\") %>%\n update_role_requirements(role = \"id\", bake = TRUE) %>% \n step_zv(all_nominal_predictors())" }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#isomap-pros-and-cons", - "href": "slides/advanced-05-feature-engineering-part-one.html#isomap-pros-and-cons", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "Isomap Pros and Cons", - "text": "Isomap Pros and Cons\n\n\nPros\n\nCaptures non-linear effects\nCaptures long-range structure, not just local structure\nNo parameters to set other than neighbors\n\n\nCons\n\nComputationally expensive\nAssumes a single connected manifold" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---handle-correlations", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#a-recipe---handle-correlations", + "title": "Case Study on Transportation", + "section": "A recipe - handle correlations ", + "text": "A recipe - handle correlations \nThe station columns have a very high degree of correlation.\nWe might want to decorrelated them with principle component analysis to help the model fits go more easily.\nThe vector stations contains all station names and can be used to identify all the relevant columns.\n\nchi_pca_rec <- \n chi_rec %>% \n step_normalize(all_of(!!stations)) %>% \n step_pca(all_of(!!stations), num_comp = tune())\n\nWe’ll tune the number of PCA components for (default) values of one to four." }, { - "objectID": "slides/advanced-05-feature-engineering-part-one.html#isomap-with-recipes", - "href": "slides/advanced-05-feature-engineering-part-one.html#isomap-with-recipes", - "title": "5 - Feature engineering: dummies and embeddings", - "section": "isomap with recipes ", - "text": "isomap with recipes \n\nlibrary(embed)\nset.seed(1234)\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_isomap(all_numeric_predictors(), neighbors = 10) |>\n prep() |>\n bake(NULL)\n#> # A tibble: 32 × 6\n#> mpg Isomap1 Isomap2 Isomap3 Isomap4 Isomap5\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 21 -98.5 35.6 35.3 137. 105. \n#> 2 21 -98.5 35.6 35.1 137. 104. \n#> 3 22.8 -161. 48.1 20.4 160. 129. \n#> 4 21.4 8.55 14.6 37.2 86.8 125. \n#> 5 18.7 136. 28.5 35.7 -30.8 102. \n#> 6 18.1 -27.7 21.7 38.3 104. 116. \n#> 7 14.3 207. -16.5 34.6 -55.5 158. \n#> 8 24.4 -147. 44.2 23.7 182. 88.4\n#> 9 22.8 -125. 41.1 18.1 151. 118. \n#> 10 19.2 -91.6 33.7 48.6 135. 101. \n#> # ℹ 22 more rows" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#make-some-models", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#make-some-models", + "title": "Case Study on Transportation", + "section": "Make some models ", + "text": "Make some models \nLet’s try three models. The first one requires the rules package (loaded earlier).\n\ncb_spec <- cubist_rules(committees = 25, neighbors = tune())\nmars_spec <- mars(prod_degree = tune()) %>% set_mode(\"regression\")\nlm_spec <- linear_reg()\n\nchi_set <- \n workflow_set(\n list(pca = chi_pca_rec, basic = chi_rec), \n list(cubist = cb_spec, mars = mars_spec, lm = lm_spec)\n ) %>% \n # Evaluate models using mean absolute errors\n option_add(metrics = metric_set(mae))\n\n\nBriefly talk about Cubist being a (sort of) boosted rule-based model and MARS being a nonlinear regression model. Both incorporate feature selection nicely." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#getting-set-up", - "href": "slides/advanced-04-feature-engineering-part-one.html#getting-set-up", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Getting set up ", - "text": "Getting set up \n\nlibrary(tidymodels)\nlibrary(embed)\nlibrary(extrasteps)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)\n\n# Load our example data for this section\n\"https://github.com/tidymodels/workshops/raw/refs/heads/2025-GMOFETML/slides/leaf_data.RData\" |> \n url() |> \n load()" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#process-them-on-the-resamples", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#process-them-on-the-resamples", + "title": "Case Study on Transportation", + "section": "Process them on the resamples", + "text": "Process them on the resamples\n\n# Set up some objects for stacking ensembles (in a few slides)\ngrid_ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)\n\nchi_res <- \n chi_set %>% \n workflow_map(\n resamples = chi_rs,\n grid = 10,\n control = grid_ctrl,\n verbose = TRUE,\n seed = 12\n )" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set", - "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Leaf data set", - "text": "Leaf data set\nSlightly modified version of modeldata::leaf_id_flavia.\n\nglimpse(leaf_data)\n#> Rows: 1,907\n#> Columns: 55\n#> $ species <fct> chinese_redbud, chinese_redbud, chinese_red…\n#> $ apex <fct> none, none, none, none, none, none, none, n…\n#> $ base <fct> none, none, none, none, none, none, none, n…\n#> $ shape <fct> heart_shape, heart_shape, heart_shape, hear…\n#> $ edge <chr> \"smooth\", \"smooth\", \"smooth\", \"smooth\", \"sm…\n#> $ outlying_polar <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ skewed_polar <dbl> 0.4610066, 0.4747877, 0.5356883, 0.4771338,…\n#> $ clumpy_polar <dbl> 0.006479784, 0.011100482, 0.017317486, 0.01…\n#> $ sparse_polar <dbl> 0.01449941, 0.01451566, 0.02953578, 0.01425…\n#> $ striated_polar <dbl> 0.9788360, 0.9797980, 0.6758621, 0.9896373,…\n#> $ convex_polar <dbl> 0.000899987, 0.000152532, 0.035230741, 0.00…\n#> $ skinny_polar <dbl> 0.1177954, 0.5440493, 0.7764908, 0.7067394,…\n#> $ stringy_polar <dbl> 1.0000000, 1.0000000, 0.8544140, 1.0000000,…\n#> $ monotonic_polar <dbl> 0.026807610, 0.005554220, 0.068481538, 0.09…\n#> $ outlying_contour <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…\n#> $ skewed_contour <dbl> 0.4954035, 0.5321818, 0.5451171, 0.4675013,…\n#> $ clumpy_contour <dbl> 0.007898114, 0.010846498, 0.020271493, 0.01…\n#> $ sparse_contour <dbl> 0.01449941, 0.01436114, 0.02903878, 0.01451…\n#> $ striated_contour <dbl> 0.9744898, 0.9811321, 0.6766917, 0.9784946,…\n#> $ convex_contour <dbl> 0.000479470, 0.000000000, 0.035908221, 0.00…\n#> $ skinny_contour <dbl> 0.1810093, 1.0000000, 0.7806988, 0.1348225,…\n#> $ stringy_contour <dbl> 1.0000000, 1.0000000, 0.8864787, 1.0000000,…\n#> $ monotonic_contour <dbl> 0.000816260, 0.000008740, 0.001253195, 0.32…\n#> $ num_max_points <dbl> 1, 3, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 6, 1…\n#> $ num_min_points <dbl> 2, 4, 7, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 8, 2…\n#> $ diameter <dbl> 1255.830, 1227.826, 1113.228, 1219.185, 118…\n#> $ area <dbl> 936626.5, 917300.5, 855376.5, 901303.0, 885…\n#> $ perimeter <dbl> 3830.534, 3781.445, 3695.646, 3678.409, 383…\n#> $ physiological_length <dbl> 1253, 1218, 1098, 1205, 1153, 1179, 1188, 1…\n#> $ physiological_width <dbl> 1088, 1091, 1083, 1053, 1078, 1073, 1054, 1…\n#> $ aspect_ratio <dbl> 0.8683160, 0.8957307, 0.9863388, 0.8738589,…\n#> $ rectangularity <dbl> 0.6870470, 0.6903027, 0.7193273, 0.7103222,…\n#> $ circularity <dbl> 0.8021540, 0.8061315, 0.7870211, 0.8370680,…\n#> $ compactness <dbl> 15.66578, 15.58849, 15.96701, 15.01237, 16.…\n#> $ narrow_factor <dbl> 1.154255, 1.125413, 1.027912, 1.157821, 1.1…\n#> $ perimeter_ratio_diameter <dbl> 3.050202, 3.079790, 3.319756, 3.017104, 3.2…\n#> $ perimeter_ratio_length <dbl> 3.520711, 3.466036, 3.412416, 3.493266, 3.5…\n#> $ perimeter_ratio_lw <dbl> 1.636281, 1.637698, 1.694473, 1.629056, 1.7…\n#> $ num_convex_points <dbl> 125, 128, 114, 138, 125, 125, 121, 112, 115…\n#> $ perimeter_convexity <dbl> 0.9404967, 0.9393550, 0.9232549, 0.9502414,…\n#> $ area_convexity <dbl> 0.02359372, 0.03175513, 0.03932713, 0.01740…\n#> $ area_ratio_convexity <dbl> 0.9769501, 0.9692222, 0.9621610, 0.9828892,…\n#> $ equivalent_diameter <dbl> 1092.0393, 1080.7142, 1043.5992, 1071.2491,…\n#> $ eccentricity <dbl> 0.44400376, 0.26866788, 0.31952502, 0.44811…\n#> $ contrast <dbl> 38.69163, 32.73154, 23.20849, 30.14546, 23.…\n#> $ correlation_texture <dbl> 0.9959609, 0.9967740, 0.9975354, 0.9968981,…\n#> $ inverse_difference_moments <dbl> 0.5951133, 0.6022763, 0.6363508, 0.6135506,…\n#> $ entropy <dbl> 6.291552, 6.273267, 5.764995, 6.203471, 5.9…\n#> $ mean_red_val <dbl> 38.02429, 34.55425, 30.92509, 33.44151, 33.…\n#> $ mean_green_val <dbl> 72.38449, 69.89448, 69.22250, 71.32743, 70.…\n#> $ mean_blue_val <dbl> 42.78902, 42.93044, 40.80089, 40.73282, 42.…\n#> $ std_red_val <dbl> 41.66486, 40.48649, 38.55110, 39.09408, 40.…\n#> $ std_green_val <dbl> 74.26665, 73.05275, 76.77596, 75.85529, 75.…\n#> $ std_blue_val <dbl> 46.36232, 48.04195, 48.09177, 46.57170, 48.…\n#> $ correlation <dbl> -0.027629688, -0.003103965, -0.036341630, 0…" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#how-do-the-results-look", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#how-do-the-results-look", + "title": "Case Study on Transportation", + "section": "How do the results look?", + "text": "How do the results look?\n\nrank_results(chi_res)\n#> # A tibble: 31 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 pca_cubist Preprocessor1_Model1 mae 0.806 0.104 16 recipe cubis… 1\n#> 2 pca_cubist Preprocessor3_Model3 mae 0.990 0.114 16 recipe cubis… 2\n#> 3 pca_cubist Preprocessor4_Model2 mae 0.990 0.121 16 recipe cubis… 3\n#> 4 pca_cubist Preprocessor4_Model1 mae 1.00 0.127 16 recipe cubis… 4\n#> 5 pca_cubist Preprocessor3_Model2 mae 1.00 0.119 16 recipe cubis… 5\n#> 6 pca_cubist Preprocessor2_Model2 mae 1.02 0.118 16 recipe cubis… 6\n#> 7 pca_cubist Preprocessor1_Model3 mae 1.05 0.131 16 recipe cubis… 7\n#> 8 basic_cubist Preprocessor1_Model8 mae 1.07 0.115 16 recipe cubis… 8\n#> 9 basic_cubist Preprocessor1_Model7 mae 1.07 0.112 16 recipe cubis… 9\n#> 10 basic_cubist Preprocessor1_Model6 mae 1.07 0.114 16 recipe cubis… 10\n#> # ℹ 21 more rows" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Leaf data set", - "text": "Leaf data set\n\n\nLakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077." + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#plot-the-results", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#plot-the-results", + "title": "Case Study on Transportation", + "section": "Plot the results ", + "text": "Plot the results \n\nautoplot(chi_res)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-2", - "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-2", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Leaf data set", - "text": "Leaf data set\n\n\nLakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077." + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#pull-out-specific-results", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#pull-out-specific-results", + "title": "Case Study on Transportation", + "section": "Pull out specific results ", + "text": "Pull out specific results \nWe can also pull out the specific tuning results and look at them:\n\nchi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n autoplot()" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-3", - "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-data-set-3", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Leaf data set", - "text": "Leaf data set\n\n\nLakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077." + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit? \nModel stacks generate predictions that are informed by several models." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#your-turn", - "href": "slides/advanced-04-feature-engineering-part-one.html#your-turn", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Your turn", - "text": "Your turn\n\nLoad and explore the leaf data\nMake time to look at the edge column and think about how one would encode it into numerics\n\n\n\n−+\n05:00" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-1", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-1", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-edges", - "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-edges", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Leaf edges", - "text": "Leaf edges\n\n\n\nLow number of unique levels\nSome overlap\n\n\n\nleaf_data |>\n count(edge)\n#> # A tibble: 7 × 2\n#> edge n\n#> <chr> <int>\n#> 1 \"\" 62\n#> 2 \"denate\" 65\n#> 3 \"lobed\" 109\n#> 4 \"lobed, smooth\" 52\n#> 5 \"lobed, toothed\" 109\n#> 6 \"smooth\" 1204\n#> 7 \"toothed\" 306" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-2", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-2", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#leaf-edges-dummies", - "href": "slides/advanced-04-feature-engineering-part-one.html#leaf-edges-dummies", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Leaf edges dummies", - "text": "Leaf edges dummies\n(technically one-hot encoding)\n\n\n\n\n\n\nX\ndenate\nlobed\nlobed..smooth\nlobed..toothed\nsmooth\ntoothed\n\n\n\n\n0\n0\n0\n0\n1\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n1\n\n\n0\n0\n0\n0\n0\n1\n0\n\n\n0\n0\n0\n0\n0\n1\n0\n\n\n0\n0\n0\n0\n0\n0\n1\n\n\n0\n0\n1\n0\n0\n0\n0" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-3", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-3", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#dummy-variables", - "href": "slides/advanced-04-feature-engineering-part-one.html#dummy-variables", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Dummy variables", - "text": "Dummy variables\n\n\nPros\n\nCommonly used\nEasy interpretation\nWill rarely lead to a decrease in performance\n\n\nCons\n\nCan create many columns\nNeeds clean levels\nNot likely the most efficient representation" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-4", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-4", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#overlapping", - "href": "slides/advanced-04-feature-engineering-part-one.html#overlapping", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Overlapping", - "text": "Overlapping\n\n\nSome of the labels\n\nlobed\nsmooth\ntoothed\nlobed, smooth\nlobed, toothed\n\n\nIf you want to find a “toothed” leaf, which level do you pick?\n\ntoothed\nlobed, toothed\ntoothed and lobed, toothed\n\n\nWe can let lobed, toothed be counted for lobed and toothed" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-5", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#why-choose-just-one-final_fit-5", + "title": "Case Study on Transportation", + "section": "Why choose just one final_fit? ", + "text": "Why choose just one final_fit?" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies", - "text": "Advanced Dummies\nThis is basically a poor man’s Natural Language Processing.\n\n\ntokenization -> counting\n\n\nWe think it is a frequent enough case that it is considered its own method.\nWe have 2 variants: “extraction” and “multi choice”" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#building-a-model-stack", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#building-a-model-stack", + "title": "Case Study on Transportation", + "section": "Building a model stack ", + "text": "Building a model stack \n\nlibrary(stacks)\n\n\nDefine candidate members\nInitialize a data stack object\nAdd candidate ensemble members to the data stack\nEvaluate how to combine their predictions\nFit candidate ensemble members with non-zero stacking coefficients\nPredict on new data!" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction", - "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\nWorks on a singular column, using a regular expression to extract the items we want to count.\nDone using regular expressions, either by specifying sep to split the string by, or by using pattern to extract the items.\nAllows for 0 to many items in each string.\nImplemented as step_dummy_extract().\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#start-the-stack-and-add-members", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#start-the-stack-and-add-members", + "title": "Case Study on Transportation", + "section": "Start the stack and add members ", + "text": "Start the stack and add members \nCollect all of the resampling results for all model configurations.\n\nchi_stack <- \n stacks() %>% \n add_candidates(chi_res)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"\"\n\"denate\"\n\"lobed\"\n\"lobed, smooth\"\n\"lobed, toothed\"\n\"smooth\"\n\"toothed\"\n\n\nUsing\nsep = \", \"\n\nResult\n\n\n\n\n\n\ndenate\nlobed\nsmooth\ntoothed\n\n\n\n\n0\n0\n0\n0\n\n\n1\n0\n0\n0\n\n\n0\n1\n0\n0\n\n\n0\n1\n1\n0\n\n\n0\n1\n0\n1\n\n\n0\n0\n1\n0\n\n\n0\n0\n0\n1" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#estimate-weights-for-each-candidate", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#estimate-weights-for-each-candidate", + "title": "Case Study on Transportation", + "section": "Estimate weights for each candidate ", + "text": "Estimate weights for each candidate \nWhich configurations should be retained? Uses a penalized linear model:\n\nset.seed(122)\nchi_stack_res <- blend_predictions(chi_stack)\n\nchi_stack_res\n#> # A tibble: 4 × 3\n#> member type weight\n#> <chr> <chr> <dbl>\n#> 1 pca_cubist_1_1 cubist_rules 0.410\n#> 2 basic_cubist_1_4 cubist_rules 0.266\n#> 3 pca_cubist_3_2 cubist_rules 0.193\n#> 4 pca_lm_4_1 linear_reg 0.171" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-2", - "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-2", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"Etching on paper\"\n\"Oil paint on canvas\"\n\"Acrylic paint on paper\"\n\"Oil paint and wax on canvas\"\n\"Oil paint, ink on canvas\"\n\n\nUsing\nsep = \", \"\n\nResult\n\n\n\n\n\n\n\n\n\n\n\n\n\nAcrylic.paint.on.paper\nEtching.on.paper\nink.on.canvas\nOil.paint\nOil.paint.and.wax.on.canvas\nOil.paint.on.canvas\n\n\n\n\n0\n1\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n1\n\n\n1\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n1\n0\n\n\n0\n0\n1\n1\n0\n0" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#how-did-it-do", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#how-did-it-do", + "title": "Case Study on Transportation", + "section": "How did it do? ", + "text": "How did it do? \nThe overall results of the penalized model:\n\nautoplot(chi_stack_res)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-3", - "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-3", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"Etching on paper\"\n\"Oil paint on canvas\"\n\"Acrylic paint on paper\"\n\"Oil paint and wax on canvas\"\n\"Oil paint, ink on canvas\"\n\n\nUsing\nsep = \"(, )|( and )|( on )\"\n\nResult\n\n\n\n\n\nAcrylic.paint\ncanvas\nEtching\nink\nOil.paint\npaper\nwax\n\n\n\n\n0\n0\n1\n0\n0\n1\n0\n\n\n0\n1\n0\n0\n1\n0\n0\n\n\n1\n0\n0\n0\n0\n1\n0\n\n\n0\n1\n0\n0\n1\n0\n1\n\n\n0\n1\n0\n1\n1\n0\n0" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#what-does-it-use", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#what-does-it-use", + "title": "Case Study on Transportation", + "section": "What does it use? ", + "text": "What does it use? \n\nautoplot(chi_stack_res, type = \"weights\")" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-4", - "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---extraction-4", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Extraction", - "text": "Advanced Dummies - Extraction\n\n\nInput\n\n\"['red', 'blue']\"\n\"['red', 'blue', 'white']\"\n\"['blue', 'blue', 'blue']\"\n\n\nUsing\npattern = \"(?<=')[^',]+(?=')\"\n\nResult\n\n\n\n\n\nblue\nred\nwhite\n\n\n\n\n1\n1\n0\n\n\n1\n1\n1\n\n\n3\n0\n0" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#fit-the-required-candidate-models", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#fit-the-required-candidate-models", + "title": "Case Study on Transportation", + "section": "Fit the required candidate models", + "text": "Fit the required candidate models\nFor each model we retain in the stack, we need their model fit on the entire training set.\n\nchi_stack_res <- fit_members(chi_stack_res)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---multi-choice", - "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---multi-choice", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Multi Choice", - "text": "Advanced Dummies - Multi Choice\nWorks on multiple columns, counting items across columns.\nIt can be seen as joining multiple applications of dummy variables together.\nAllows for 0 to many items.\nImplemented as step_dummy_multi_choice().\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#the-test-set-best-cubist-model", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#the-test-set-best-cubist-model", + "title": "Case Study on Transportation", + "section": "The test set: best Cubist model ", + "text": "The test set: best Cubist model \nWe can pull out the results and the workflow to fit the single best cubist model.\n\nbest_cubist <- \n chi_res %>% \n extract_workflow_set_result(\"pca_cubist\") %>% \n select_best()\n\ncubist_res <- \n chi_res %>% \n extract_workflow(\"pca_cubist\") %>% \n finalize_workflow(best_cubist) %>% \n last_fit(split = chi_split, metrics = metric_set(mae))" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---multi-choice-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#advanced-dummies---multi-choice-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Advanced Dummies - Multi Choice", - "text": "Advanced Dummies - Multi Choice\n\n\nInput\n\n\n\n\n\nlang_1\nlang_2\nlang_3\n\n\n\n\nEnglish\nNA\nItalian\n\n\nSpanish\nFrench\nFrench\n\n\nNA\nNA\nNA\n\n\nEnglish\nNA\nNA\n\n\n\n\n\n\nResult\n\n\n\n\n\nEnglish\nFrench\nItalian\nSpanish\n\n\n\n\n1\n0\n1\n0\n\n\n0\n1\n0\n1\n\n\n0\n0\n0\n0\n\n\n1\n0\n0\n0" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#the-test-set-stack-ensemble", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#the-test-set-stack-ensemble", + "title": "Case Study on Transportation", + "section": "The test set: stack ensemble", + "text": "The test set: stack ensemble\nWe don’t have last_fit() for stacks (yet) so we manually make predictions.\n\nstack_pred <- \n predict(chi_stack_res, chi_test) %>% \n bind_cols(chi_test)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#othering", - "href": "slides/advanced-04-feature-engineering-part-one.html#othering", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Othering", - "text": "Othering\nBoth step_dummy_extract() and step_dummy_multi_choice() contain a threshold argument.\nThis is used to combine infrequent levels together.\nWe have a step step_other() that does this for nominal variables, but it doesn’t work in these cases, as it has to happen after the extraction/combination." + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#compare-the-results", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#compare-the-results", + "title": "Case Study on Transportation", + "section": "Compare the results ", + "text": "Compare the results \nSingle best versus the stack:\n\ncollect_metrics(cubist_res)\n#> # A tibble: 1 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 mae standard 0.670 Preprocessor1_Model1\n\nstack_pred %>% mae(ridership, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 mae standard 0.687" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#othering-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#othering-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Othering", - "text": "Othering\nthreshold = 0 produces 1217 columns.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nX1.monitor\nX1.person\nX1.projection\nX10.light.boxes\nX10.tranformers\nX100.digital.prints\nX100.works\nX11.photographs\nX11.works\nX12.ceramic.tea.mugs\nX12.drawings\nX13.canvases\nX133.slides\nX14.monitors\nX14.photographs\nX14.small.cubes\nX14.works\nX15.engraved.plaques\nX15.hand.coloured.photographs\nX15.photographs\nX15.rooms\nX15.steel.kitchen.ustensils\nX15.unglazed.ceramic.forms\nX151.text.panels\nX16.channels\nX16.mm\nX16.paintings\nX16.sheets\nX160.slides\nX16mm\nX18.drawings\nX18.photographs\nX19.wooden.pieces.of.furniture\nX2.aluminium.panels\nX2.aluminium.tables\nX2.back.projections\nX2.banners\nX2.canvases\nX2.car.batteries\nX2.copper.panels\nX2.digital.prints\nX2.electric.cables\nX2.flat.screens\nX2.fluorescent.tubes\nX2.hooks\nX2.horses\nX2.light.bulbs\nX2.lithographs\nX2.maps\nX2.marker.pens\nX2.metal.scaffold.towers\nX2.monitors\nX2.paintings\nX2.papers\nX2.people\nX2.photographs\nX2.porcelain.light.sockets\nX2.projections\nX2.ropes\nX2.screenprints\nX2.sections.of.pre.cast.concrete.pipe\nX2.sheets\nX2.slippers\nX2.stainless.steel.discs\nX2.trains\nX2.transparencies\nX2.videos\nX2.wall.pieces\nX2.works\nX20.cotton.mattresses\nX20.flat.screens.or.1.projection\nX20.photographs\nX20.wooden.beds\nX2000.gerberas\nX201.photographs\nX21.aluminium.bricks\nX21.channels\nX21.works\nX22.photographs\nX220.works\nX23.monitors\nX24.light.bulbs\nX24.works\nX25.photographs\nX26.works\nX27.photographs\nX28.etchings\nX3.balls\nX3.books\nX3.canvases\nX3.flat.screens\nX3.Kentia.palms.in.3.terracotta.pots\nX3.lamps\nX3.light.bulbs\nX3.monitors\nX3.people\nX3.photographs\nX3.printers\nX3.prints\nX3.projections\nX3.screenprints\nX3.skateboards\nX3.transformers\nX3.wall.mounted.LCD.monitors\nX3.works\nX30.light.emitting.diodes.units\nX30.works\nX31.photographs\nX32.digital.prints\nX32.flat.screens\nX32.papers\nX320.slides\nX34.photographs\nX34.wooden.sculptures\nX35.mm\nX35.works\nX350.digital.prints\nX355.photographs\nX35mm\nX36.tin.dogs\nX39.cardboard.boxes\nX39.metronomes\nX4.aluminium.pixel.boxes.with.DMX.control.box\nX4.Belisha.beacons\nX4.canvases\nX4.channels\nX4.corrugated.slabs.of.concrete\nX4.digital.prints\nX4.digital.prints.with.acrylic.paint\nX4.fibreboard.panels\nX4.flat.screens\nX4.gascookers\nX4.hardboard.walls\nX4.light.bulbs\nX4.mannequins\nX4.monitors\nX4.photocopies\nX4.photographs\nX4.projections\nX4.steel.capsules\nX40.light.emitting.diode.units\nX40.photographs\nX40.works\nX42.electric.lamps\nX42.photographs\nX46.photographs\nX49.Perspex.boxes\nX5.digital.prints\nX5.drawings\nX5.flat.screens\nX5.knives\nX5.monitors\nX5.painted.plaster.busts\nX5.painted.wooden.shelves\nX5.photographs\nX5.projections\nX5.tables\nX5.works\nX50.canvases\nX50.cardboard.boxes\nX50.slides\nX53.photographs\nX54.digital.prints\nX6.digital.prints\nX6.fabrics\nX6.lithographs\nX6.monitors\nX6.Perspex.panels\nX6.photographs\nX6.projections\nX6.works\nX60.digital.prints\nX60.works\nX62.florescent.lights\nX64.tin.soldiers\nX7.channels\nX7.drawings\nX7.light.emitting.diode.columns\nX7.panels\nX7.projections\nX7.stools\nX7.works\nX71.vinyl.records\nX76.works\nX8.channels\nX8.digital.prints\nX8.headphones\nX8.letterpress.prints\nX8.mm\nX8.monitors\nX8.projections\nX8.synthetic.fabric.flags\nX8.woodblock\nX8.works\nX80.slides\nX800.digital.prints\nX84.sleeves\nX9.doors\nX9.LED.lights\nX9.photographs\nX9.works\nX90.photographs\nacetate\nacrylic\nacrylic.fibre\nacrylic.glass\nacrylic.pai\nacrylic.paint\nAcrylic.paint\nacrylic.resin\nacrylic.sheet\nAcrylic.sheet\nacrylic.sheets\nAcrylic.tubes\nadhesive\nadhesive.tape\naerials\nAgfa.Isolette.camera\nAlabaster\nalarm.clocks\nAlkyd.paint\nAltered.Eames.plywood.leg.splints\naluminium\nAluminium\naluminium.buckles\naluminium.disc\naluminium.lightbox\nAluminium.machinery.part\naluminium.paint\naluminium.panel\naluminuim\nAluminuim\naluminum.panel\namplifier\nand.sound\nand.sound..mono\nand.sound..mono.\nand.sound..stereo.\nanimal.bones\nAnimal.intestines\naquatint\nAquatint\naquatint.with.hand.colouring\nartificial.foliage\nartificial.moss\nartificial.sand\nartificial.wig\nartist.s.hair\nash\nAsh\nashtray\naudio\nAudio\naudio.system\naudiotrack\nback.projection\nbadge\nbadges.with.printed.papers\nballasts\nbalsa.wood\nBamboo.cane\nBamboo.poles\nbark\nbaseball.cap\nbath.bombs\nbatteri\nbauxite.rocks\nbeads\nBeanbag\nbeech\nbeer.can\nbells\nbicycle.belt\nbindis\nbins\nblack\nblackboard\nBlind.embossed.print\nboard\nBoard\nBoat\nbone\nbones\nbook\nBook\nbook.cover\nbooklets\nbooks\nboots\nbowl\nbox\nbrass\nBrass\nbrass.pins\nbrass.plate\nbreeze.blocks\nbrick\nBriefcase\nbronze\nBronze\nBronze.bells\nbronze.powder\nBronze.with.silk.scarf\nBronze..with.Jamie.Sargeant\nBronze..with.Nicholas.Sloan\nbrooms\nbuckets\nbuckram\nbuoy\nburlaq.sack\nburnt.wood\nButterflies\nbuttons\ncable\ncables\ncalf\ncalico\ncandle\ncanvas\nCanvas\nCanvas.lining.with.ingrained.dust\ncanvas.paper\nCanvas.tacking.edges\ncarbon.fibre\ncarborundum\ncarborundum.mezzotint\ncard\nCard\ncardboard\nCardboard\ncardboard.architectural.model\ncardboard.base\ncardboard.box\nCardboard.box\ncardboard.boxes\ncardboard.coffin\ncarpet\nCarpet\ncarpets\nCarrara.marble\nCast.iron\ncastor.wheels\nCedar.wood\nceiling\nCellophane\ncellulose\ncellulose.lacquer\ncellulose.print\ncement\ncement.blocks\ncentral.processing.unit\nceramic\nCeramic\nceramic.tiles\nchain\nchair\nchair.parts\nchairs\nchalk\nChalk\ncharcoal\nCharcoal\nchine.collé\nchipboard\nChipboard\nchrome\nchrome.steel.balls\nChromogenic.print\ncibachrome\ncibachrome.print\nCibachrome.print\ncigarette.ash\ncigarette.boxes\ncigarette.butts\ncigarette.sheets\ncigarettes\nclay\nClay\nclothes\ncloves\ncoal\ncoat\nCoca.cola.bottle\ncoconut.oil\ncoffee\ncoin\ncollage\ncollagraph\ncolour\ncolour.and\ncolour.ans.sound..stereo.\ncoloured.graphite\ncoloured.light.bulbs\ncoloured.pencil\ncoloured.pencils\ncolumns\nCommercial.paint\ncompressor\ncompressors\ncomputer\nConcrete\nConté\nContractor.s.shed\nCooked.couscous\ncooker\ncopper\ncopper.sulphate\ncopperplate\ncopperplated\ncoral\ncord\nCork.panels\ncorrection.fluid\nCorrection.fluid\ncotton\nCotton\ncotton.costume\ncotton.thread\ncotton.wool\ncow\nCowhide\ncrayon\nCrayon\ncrude.oil\ncrystalline.particles\ncue.stand\ncues\ncurtains\nCut.paper\nDelabole.slate\ndesks\ndetritus\ndigging.tools\ndigital.image\ndigital.print\nDigital.print\nDigital.print.with.acrylic.paint\ndigital.prints\nDigital.prints.with.acrylic.paint\ndolls\ndoor\ndrawings\nDresden.D1.projector\ndress.shirt\ndry.pigment.paint\ndrypoint\nDrypoint\nduck.tape\ndye\nearth\nEarth\nearthenware\nEarthenware\nelectric.cable\nelectric.wire\nelectrical.a\nelectrical.cable\nelectrical.components\nelectrical.pump\nelectronic.circuit\nelephant.dung\nemulsion\nEmulsion\nenamel\nEnamel\nenamel.button\nenamel.paint\nEnamel.paint\nencre.de.Chine\nEngine\nengraving\nEngraving\nenvelopes\nepoxy\netching\nEtching\nEtching.with.hand.colouring\nexpanding.foam\neyebrow.pencil\neyeshadow\nfabric\nFabric\nfabric.patches\nfaced.particleboard\nfan.heater\nFeathers\nFed.2.type.camera\nfelt\nFelt\nfelt.pen\nFerric.oxide\nfibre.clay\nfibreboard\nFibreboard\nFibreboard.cabinet\nfibreboard.panels\nfibreboard.plinth\nfibreboard.with.clay\nfibreglass\nFibreglass\nfilm\nFilm\nFilm.16.mm\nflat.screen\nflat.screen.or.projector\nflax\nFlies\nFlint\nfloodlight\nflowers\nfluorescent.light\nfluorescent.lights\nfluorescent.site.jacket\nfluorescent.tubes\nflyers\nfoam\nfoam.core\nfoam.rubber\nfoil\nfoil.tape\nfood\nformaldehyde.solution\nformica\nFormica\nfound.posters\nframed.\nFuton.mattress\nGallery.lighting\ngalvanised.steel\ngalvanized.steel\ngel\ngelatin.silver.print\ngelatin.silver.printon.paper\ngelatine.silver.print.with.dye\nglas\nglass\nGlass\nglass.a\nglass.beads\nglass.box\nGlass.chandelier\nglass.vitrine.containing.beauty.and\nGlass.10.hardbacked.books\nglitter\nglove\nglue\ngobo\ngold.leaf\ngold.metallic.paint\nGold.paint\ngouache\nGouache\nGranite\ngraphite\nGraphite\ngrill\nGrpahite\nguitar.strings\ngun\nGunpowder\nhair\nhair.over.metal\nHand.coloured.photograph\nhand.colouring\nHand.cut.book\nhandcolouring\nhardboard\nhardboard.plinth\nhardboard.table\nhardwood\nhat\nheadphones\nHeart.to.Hear\nHeavy.goods.vehicle\nhelium.balloon\nhemp.cord\nhessian\nHessian\nhigh.definition\nhigh.definition_2\nhook\nhorn\nhosepipes\nHousehold.emulsion.paint\nhousehold.gloss.paint\nhousehold.paint\nHousehold.paint\nhuman.hair\nhydraulic.rams\nidentity.card\ninfrared.sensors\ningrained.dust\nink\nInk\ninteractive\nIris.print\niron\niron.powder\njacket\nJacket\njute\nkapok\nkey.\nKilkenny.limestone\nkitchen.utensils\nkite\nknife\nl\nlab.coat\nlacquer\nLacquered.wood\nladder\nlamb\nLambda.print\nlamps\nLamps\nlatex\nlead\nLead\nleather\nLeather\nleather.briefcase\nletterpress\nLetterpress\nlice\nlight\nlight.box\nlight.boxes\nlight.bulb\nlight.bulbs\nlight.control.unit\nlightbox\nLightboxes\nlighter\nlighting.system\nlights\nlinen\nlinocut\nLinocut\nLinocut.print\nlinoleum\nlithograph\nLithograph\nlithograph.with.plasticine\nLithograph.woodcut\nlithographs\nmagnetic.rods\nmagnets\nMahogany\nmango.seeds\nmannequin\nMannequin\nmap\nMap\nmap.pins\nmarble\nMarble\nmarble.base\nmarker.pen\nMarker.pen\nMDF\nMDF.backboard\nme\nmelamine\nmelanine.fibreboard\nMelinex\nmetal\nMetal\nMetal.bicycle\nmetal.bottle.caps\nmetal.bucket\nmetal.chain\nmetal.clamp\nmetal.clothes.rail\nMetal.cot\nmetal.detritus\nmetal.film.spool\nmetal.foil\nMetal.frame\nMetal.frames\nmetal.hook\nmetal.pipe\nmetal.sheet\nmetal.string\nMetal.table\nMetal.tin.cans\nmetal.wire\nmetalcut.prints\nmetallic.powder\nmezzotint\nMezzotint\nmica.flakes\nmicrophone\nmin.DV.cam.tape\nmini.disc.player\nmirror\nMirror\nmirrors\nMiss.Piggy.Bag..and.contents.\nMixed.media\nmobile.pho\nmodelling.putty\nmonitor\nmonitor.or.flat.screen\nmonitor.or.projection\nmono\nMonoprint\nmonotype\nMonotype\nmorse.code.unit\nmotor\nmotorised.base\nmoun\nMountain.bicycle\nMountain.peak.in.container\nmountaineering.equipment\nmounted\nmounted.onto.aluminium\nMud\nmultiple.projections\nmuslin\nMylar.screen\nnails\nnatural.fibres\nneon.gas\nneon.lights\nNeon.lights\nneon.paint\nnewspapers\nnotebook\nnylon\nnylon.straps\nnylon.strings\nNylon.tights\nnyloprints\noak\nOak\noak.table\noak.twig\noffice\noil\noil.paint\nOil.paint\noil.pastel\noil.stick\nOil.stick\noil.tint\non.47.panels\non.aluminium\non.aluminium.panel\non.board\non.cardboard\non.paper\non.paper.between.glass\non.paper.mounted\non.paper.mounted.onto.acrylic.glass\non.paper.mounted.onto.aluminium\non.paper.mounted.onto.aluminium.panel\non.paper.mounted.onto.aluminuim\non.paper.mounted.onto.board\non.paper.mounted.onto.panel\non.paper.mounted.onto.paper\non.paper.mounted.onto.Perspex\non.paper.mounted.onto.plastic\non.paper.with.chalk\non.paper.with.dry.transfert.print.mounted.onto.paper\non.paper.with.paint\non.papers\non.papers.with.paint\non.plastic\non.sticker.paper\non.vinyl.mounted.onto.aluminium\nopen.cell.foam\noptical.gelatin.silver.fibre.print\nor.video\nOrgan\nOstrich.egg\nother.m\nother.materia\nother.materials\nothers.materials\noxidised.brass\np\npaint\nPaint\nPaint.brush\npainted.aluminium\nPainted.aluminium\npainted.boards\nPainted.fibreboard\npainted.glass\npainted.MDF\nPainted.steel\npainted.wall\npainted.wall.text\npainted.wood\nPainted.wood\nPainted.wooden.building.with.asphalt.shingle.roof\npalm.fronds\nPalm.tree\npanel\npaper\nPaper\npaper.mounted\npaper.mounted.onto.aluminium\npaper.mounted.onto.aluminium.panel\npaper.mounted.onto.board\npaper.mounted.onto.foam.core\npaper.mounted.onto.muslin\npaper.mounted.onto.panel\npaper.mounted.onto.paper\npaper.pulp\npaper.tape\npaper.with.dry.transfer.print\npaper.with.dye\npaper.with.ink\npaper.with.oil.paint\npaper.with.watercolour\npaper..Verso..ink\npapers\nparaffin\nparaffin.lamp\npassport\npastel\nPastel\npatina\npeacock.feathers\npen\npendant.lamps\npeople\nPerformance\nperfume.flask.with.pouch\nperspex\nPerspex\nPewter\npharmaceutical.packaging\nphoto.etching\nPhoto.etching\nphotograph\nPhotograph\nphotograph.mounted\nPhotographic.contact.sheet\nphotographs\nPhotographs\nphotographs.\nPiano\npigment\nPigment\nPigment.transfer\npigments\nPine.plywood\npins\nplants\nplaster\nPlaster\nplaster.figures\nplaster.powder\nplasterboard\nplastic\nPlastic\nplastic.bag\nplastic.bags\nplastic.beads\nPlastic.beads\nplastic.box\nplastic.boxes\nplastic.cover\nPlastic.lids\nplastic.pearls\nplastic.pen.lids\nplastic.pillow\nplastic.shoe\nplastic.thread\nplastic.tubes\nplastic.watch.with.photograph\nplastic.watering.can\nplastic.watering.can..spray.bot\nplasticine\nPlasticine\nplatinium.print\nPlexiglas\nplexiglass\nplinth\nplug.boards\nplywood\nPlywood\nplywood.board\npo\nPolychromed.aluminium\nPolyester\npolyester.foam\npolyester.resin\nPolyester.resin\npolyester.textile\npolyfibre\npolymer\npolystyrene\npolystyrene.foam\npolythene\npolyurethane\npolyurethane.foam\nPolyurethane.resin\nPolyurethane.rubber\npolyvinyl.acetate.paint\nPolyvinyl.acetate.paint\nporcelain\nPorcelain\nporcelain.model\nporcelaine\nportable.keyboard.keys\nPortorino.marble\npostcard\nPostcard\npostcards\nposters\nPotato.print\nPowder.coated.aluminium\npowder.coated.steel\nPowder.coated.steel\npowder.paint\npowder.coated.steel_2\nPowder.coated.steel_2\nprint\nPrint\nprinted.map\nprinted.p\nprinted.paper\nPrinted.paper\nprinted.papers\nPrinted.papers\nprojection\nprojection.or.7.monitors\nprojection.or.monitor\npumps\nPVC\nradio\nradio.transmitter\nramin\nrecipes\nRecord.deck\nrecord.player\nrecorded.voice\nreel.to.reel.tape.deck\nrelief\nRelief\nRelief.print\nresin\nResin\nresin.base\nresin.block\nribbon\nRibbon\nRiver.mud\nRoman.vessel\nrope\nropes\nrosary.beads\nrubber\nRubber\nrubber.coated.steel\nRubber.inner.tubes\nrubber.padding\nsafety.pins\nsalt\nsand\nsatin\nsatin.ribbon\nsawdust\nSax.oil.paint\nscanachrome.print\nScanachrome.print\nscissors\nscreenprint\nScreenprint\nScreenprint.with.acrylic.varnish\nScreenprints\nscrews\nsea.shells\nseat\nsection.of.concrete.wall\nsellotape\nSellotape\nSequins\nsewing.machine\nsheep.excrement\nshellac\nshellac.resin\nshells\nShells\nshield\nshirt\nshoeboxes\nshoelace\nshoes\nShop.mannequin\nshown.as\nshown.as.video\nshredder\nsilicon.hose\nsilicon.tubing\nsilicone\nsilicone.adhesive\nsilicone.rubber\nSilicone.rubber\nsilk.tie\nsilkcreen.print\nsilkscreen\nSilkscreen\nSilkscreen.print\nsilver\nsilver.foil\nsilver.gelatin.print\nsilver.solder\nSilver.teapot\nSilverpoint\nsisal.rope\nsisal.string\nSlate\nslide\nSlide\nslides\nsmall.speakers\nsmoke\nsmoke.machine\nso\nsoap\nSoap\nsocks\nSofa\nsoftware\nSoftware\nSondor.playback.machine\nsound\nsound..mono\nsound..mono.\nsound..stereo.\nsound..surround.\nsound.recording\nsound..wood\nspeakers\nspices\nspinning.top\nspray.enamel\nspray.paint\nSpray.paint\nSprayed.Q.Cell\nsreenprint\nStack.of.printed.paper\nstainless.steel\nStainless.steel\nstainless.steel.ashtray\nstainless.steel.screws\nstainless.steel.tea.urn\nstainless.steel.teapot\nstainless.steel.wire\nstand\nstaple.gun\nStarfish\nsteel\nSteel\nsteel.bars\nSteel.bedsprings\nsteel.bracket\nsteel.brake.wires\nsteel.cable\nSteel.locker\nSteel.plate\nsteel.screws\nsteel.shelving\nsteel.wire\nstencil\nstereo\nstereo.\nstickers\nstockings\nstone.base\nstone..with.Jamie.Sargeant\nstool\nstoryboards\nstraw\nstrin\nstring\nstyrene\nstyrofoam\nsugar.paper\nsuitcase\nSuper.16.mm\nSuper.8\nSuper.8.mm\nsweet.wrappers\nswing.seat\nsynthetic.fibre\nsynthetic.fibres\ntable\nTable\ntables\ntape\ntar\ntarpaulin\ntaxidermy\ntea\ntelephone\nTempera\ntent\ntextile\nTextile\ntextiles\nTextiles\ntheatre.light\nthread\nthreads\ntickets\ntights\nTimber\ntissue\ntoilet\ntooth.b\ntorchlight\ntoy.eyeballs\ntracing.paper\ntracking.system\ntram.window\ntransfer.lettering\ntransparency\nTransparency\ntransparent.paper\ntree.branches\ntripod\ntrousers\ntwigs\ntwo.pack.car.lacquer\ntype\ntypescript\ntypewritten.ink\nUHF.Radio\numbrella\nunderpants\nvarnish\nvelvet\nvelvet.p\nVHS.tape\nvideo\nVideo\nvideo.camera\nviewfinder\nvinyl\nVinyl.dispersion\nvinyl.floor.covering\nvinyl.paint\nVinyl.record\nvinyl.seat\nVinyl.tape\nvinyl.text\nvinyl.wall.text\nVinyl.wall.text\nvinyl.wall.texts\nwalkman\nWalkman\nwall\nwall.clock\nWall.painting\nwall.text\nWallpaper\nWashers\nwater\nwatercolour\nWatercolour\nwax\nweb.search.program\nwebbing\nWestern.red.cedar\nwheels\nwhite\nwire\nwire.coat.hanger.\nwires\nwood\nWood\nWood.engraving\nwood.tray\nwood.varnish\nwood.veneer\nWoodblock\nwoodcut\nWoodcut\nWooden.abacus.beads\nwooden.base\nwooden.beams\nWooden.billiard.table\nWooden.birdhouse.with.metal.roof\nwooden.boxes\nWooden.cabinet\nwooden.ceiling.props\nwooden.chair\nWooden.chair\nWooden.construction\nWooden.desk\nwooden.dowels\nwooden.floor\nwooden.frames\nwooden.glass.negative.plate.carrier\nwooden.pallets\nwooden.panel\nwooden.planks\nwooden.platform\nwooden.steps\nwooden.table\nWooden.table\nwooden.trestle\nwool\nWool\nwork.overalls\nworks\nWorry.beads\nwraps\nzinc\nother\n\n\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n\n\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0\n0" + "objectID": "archive/2024-08-posit-conf/extras-transit-case-study.html#plot-the-test-set", + "href": "archive/2024-08-posit-conf/extras-transit-case-study.html#plot-the-test-set", + "title": "Case Study on Transportation", + "section": "Plot the test set ", + "text": "Plot the test set \n\n\nlibrary(probably)\ncubist_res %>% \n collect_predictions() %>% \n ggplot(aes(ridership, .pred)) + \n geom_point(alpha = 1 / 2) + \n geom_abline(lty = 2, col = \"green\") + \n coord_obs_pred()" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#othering-2", - "href": "slides/advanced-04-feature-engineering-part-one.html#othering-2", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Othering", - "text": "Othering\nthreshold = 0.05 produces 11 columns.\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\naquatint\ncanvas\ncolour\nEtching\ngelatin.silver.print\nLithograph\non.paper\npaper\nPhotograph\nScreenprint\nother\n\n\n\n\n0\n0\n1\n0\n0\n0\n0\n0\n0\n0\n3\n\n\n0\n0\n0\n1\n0\n0\n0\n1\n0\n0\n0\n\n\n0\n0\n0\n1\n0\n0\n0\n1\n0\n0\n0\n\n\n0\n0\n0\n1\n0\n0\n0\n1\n0\n0\n0\n\n\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n1\n\n\n0\n1\n0\n0\n0\n0\n0\n0\n0\n0\n1" + "objectID": "archive/2024-08-posit-conf/index.html", + "href": "archive/2024-08-posit-conf/index.html", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels offered at posit::conf 2024. The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to tidymodels and Advanced tidymodels courses.\nIntroduction to tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nAdvanced tidymodels will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#your-turn-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#your-turn-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Your turn", - "text": "Your turn\n\nFigure out whether step_dummy_extract() or step_dummy_multi_choice() is most appropriate on the edge variable in leaf_data and apply it\n\n\n\n−+\n03:00" + "objectID": "archive/2024-08-posit-conf/index.html#welcome", + "href": "archive/2024-08-posit-conf/index.html#welcome", + "title": "Machine learning with tidymodels", + "section": "", + "text": "These are the materials for workshops on tidymodels offered at posit::conf 2024. The tidymodels framework is a collection of packages for modeling and machine learning using tidyverse principles. This website hosts the materials for both the Introduction to tidymodels and Advanced tidymodels courses.\nIntroduction to tidymodels will teach you core tidymodels packages and their uses: data splitting/resampling with rsample, model fitting with parsnip, measuring model performance with yardstick, and model optimization using the tune package. Time permitting, you’ll be introduced to basic pre-processing with recipes. You’ll learn tidymodels syntax as well as the process of predictive modeling for tabular data.\nAdvanced tidymodels will teach you about model optimization using the tune and finetune packages, including racing and iterative methods. You’ll be able to do more sophisticated feature engineering with recipes. Time permitting, model ensembles via stacking will be introduced. This course is focused on the analysis of tabular data and does not include deep learning methods." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#dimensionality-reduction-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#dimensionality-reduction-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Dimensionality Reduction", - "text": "Dimensionality Reduction\nWhat is it?\nTechniques that remove or alter features in order to have fewer but more informative features.\nWhy do we do it?\n\nredundant information\nineffective representation\ncomputational speed" + "objectID": "archive/2024-08-posit-conf/index.html#is-this-workshop-for-me", + "href": "archive/2024-08-posit-conf/index.html#is-this-workshop-for-me", + "title": "Machine learning with tidymodels", + "section": "Is this workshop for me? ", + "text": "Is this workshop for me? \nDepending on your background, one of Introduction to tidymodels or Advanced tidymodels might serve you better than the other.\n\nIntroduction to tidymodels\nThis workshop is for you if you:\n\nare comfortable using tidyverse packages to read data into R, transform and reshape data, and make a variety of graphs, and\nhave had some exposure to basic statistical concepts such as linear models, residuals, etc.\n\nIntermediate or expert familiarity with modeling or machine learning is not required. Interested students who have intermediate or expert familiarity with modeling or machine learning may be interested in the Advanced tidymodels workshop.\n\n\nAdvanced tidymodels\nThis workshop is for you if you:\n\nhave the prerequisite skills listed for the Introduction to tidymodels workshops,\nhave used tidymodels packages like recipes, rsample, and parsnip, and\nhave some experience with evaluating statistical models using resampling techniques like v-fold cross-validation or the bootstrap.\n\nParticipants who are new to tidymodels or machine learning will benefit from taking the Introduction to tidymodels workshop before joining this one. Participants who have completed the “Introduction to tidymodels” workshop will be well-prepared for this course." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#redundant-information", - "href": "slides/advanced-04-feature-engineering-part-one.html#redundant-information", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Redundant Information", - "text": "Redundant Information\nA feature with no information could be included.\n\nleaf_data |>\n count(outlying_contour)\n#> # A tibble: 1 × 2\n#> outlying_contour n\n#> <dbl> <int>\n#> 1 0 1907" + "objectID": "archive/2024-08-posit-conf/index.html#preparation", + "href": "archive/2024-08-posit-conf/index.html#preparation", + "title": "Machine learning with tidymodels", + "section": "Preparation", + "text": "Preparation\nThe process to set up your computer for either workshop will look the same. Please join the workshop with a computer that has the following installed (all available for free):\n\nA recent version of R, available at https://cran.r-project.org/\nA recent version of RStudio Desktop (RStudio Desktop Open Source License, at least v2024.04.0), available at https://posit.co/download/rstudio-desktop/\nFor all of the slides, the following R packages can be installed from the R console:\n\n\n# Install the packages for the workshop\npkgs <- \n c(\"bonsai\", \"Cubist\", \"doParallel\", \"earth\", \"embed\", \"finetune\", \n \"forested\", \"lightgbm\", \"lme4\", \"parallelly\", \"plumber\", \"probably\", \n \"ranger\", \"rpart\", \"rpart.plot\", \"rules\", \"splines2\", \"stacks\", \n \"text2vec\", \"textrecipes\", \"tidymodels\", \"vetiver\")\n\ninstall.packages(pkgs)\n\nFor the advanced course, you should install the newest version of the dials package (version 1.3.0). To check this, you can run:\n\nrlang::check_installed(\"dials\", version = \"1.3.0\")\n\nIf you’re a Windows user and encounter an error message during installation noting a missing Rtools installation, install Rtools using the installer linked here." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#redundant-information-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#redundant-information-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Redundant Information", - "text": "Redundant Information" + "objectID": "archive/2024-08-posit-conf/index.html#slides", + "href": "archive/2024-08-posit-conf/index.html#slides", + "title": "Machine learning with tidymodels", + "section": "Slides", + "text": "Slides\nThese slides are designed to use with live teaching and are published for workshop participants’ convenience. They are not meant as standalone learning materials. For that, we recommend tidymodels.org and Tidy Modeling with R.\n\nIntroduction to tidymodels\n\n01: Introduction\n02: Your data budget\n03: What makes a model?\n04: Evaluating models\n05: Tuning models\n06: Wrapping up\n\n\n\nAdvanced tidymodels\n\n01: Introduction\n02: Feature engineering using recipes\n03: Tuning hyperparameters (grid search)\n04: Grid search via racing\n05: Iterative search\n06: Wrapping up\n\n\n\nExtra content (time permitting)\n\nIntro: Using workflowsets\nIntro: Using recipes\nAdvanced: Transit case study (includes stacking)\nAdvanced: Effect encoding\nAdvanced: Model deployment\n\nThere’s also a page for slide annotations; these are extra notes for selected slides." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation", - "href": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Ineffective Representation", - "text": "Ineffective Representation\nWhile keeping our models in mind, we want to make sure the data is well-suited\n\nCorrelated data\n\nhard for some models\n\nlat/lon compared to distance/angle\n\nhard for most models" + "objectID": "archive/2024-08-posit-conf/index.html#code", + "href": "archive/2024-08-posit-conf/index.html#code", + "title": "Machine learning with tidymodels", + "section": "Code", + "text": "Code\nQuarto files for working along are available on GitHub. (Don’t worry if you haven’t used Quarto before; it will feel familiar to R Markdown users.)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Ineffective Representation", - "text": "Ineffective Representation" + "objectID": "archive/2024-08-posit-conf/index.html#past-workshops", + "href": "archive/2024-08-posit-conf/index.html#past-workshops", + "title": "Machine learning with tidymodels", + "section": "Past workshops", + "text": "Past workshops\n\nEnglish\n\nSeptember 2023 at posit::conf()\nJuly 2023 at the New York R Conference\nAugust 2022 in Reykjavik\nJuly 2022 at rstudio::conf()\n\n\n\nSpanish\n\nMarch 2024 at conectaR" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation-2", - "href": "slides/advanced-04-feature-engineering-part-one.html#ineffective-representation-2", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Ineffective Representation", - "text": "Ineffective Representation" + "objectID": "archive/2024-08-posit-conf/index.html#acknowledgments", + "href": "archive/2024-08-posit-conf/index.html#acknowledgments", + "title": "Machine learning with tidymodels", + "section": "Acknowledgments", + "text": "Acknowledgments\n\nThis website, including the slides, is made with Quarto. Please submit an issue on the GitHub repo for this workshop if you find something that could be fixed or improved." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#computational-speed", - "href": "slides/advanced-04-feature-engineering-part-one.html#computational-speed", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Computational Speed", - "text": "Computational Speed\nDepending on what method we are using and how the data is affected by it, we could see a large reduction in features. This, in turn, leads to a smaller model that is faster to train on.\nOnly exploration and trial and error can determine whether you should use dimensionality reduction techniques. Knowing which methods do what helps you determine what to try." + "objectID": "archive/2024-08-posit-conf/index.html#reuse-and-licensing", + "href": "archive/2024-08-posit-conf/index.html#reuse-and-licensing", + "title": "Machine learning with tidymodels", + "section": "Reuse and licensing", + "text": "Reuse and licensing\n\nUnless otherwise noted (i.e. not an original creation and reused from another source), these educational materials are licensed under Creative Commons Attribution CC BY-SA 4.0." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#dimensionality-reduction-method", - "href": "slides/advanced-04-feature-engineering-part-one.html#dimensionality-reduction-method", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Dimensionality Reduction Method", - "text": "Dimensionality Reduction Method\n\nZero Variance removal\nPCA\n\nTruncated PCA\nSparse PCA\n\nNNMF\nUMAP\nIsomap" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington", + "title": "2 - Your data budget", + "section": "Data on forests in Washington", + "text": "Data on forests in Washington\n\n\n\nThe U.S. Forest Service maintains ML models to predict whether a plot of land is “forested.”\nThis classification is important for all sorts of research, legislation, and land management purposes.\nPlots are typically remeasured every 10 years and this dataset contains the most recent measurement per plot.\nType ?forested to learn more about this dataset, including references.\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/251793/forest-mountain" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#restrictions", - "href": "slides/advanced-04-feature-engineering-part-one.html#restrictions", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Restrictions", - "text": "Restrictions\nAll the methods shown today will not be able to handle\n\nmissing data\nNon-numeric data" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington-1", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington-1", + "title": "2 - Your data budget", + "section": "Data on forests in Washington", + "text": "Data on forests in Washington\n\n\n\nN = 7,107 plots of land, one from each of 7,107 6000-acre hexagons in WA.\nA nominal outcome, forested, with levels \"Yes\" and \"No\", measured “on-the-ground.”\n18 remotely-sensed and easily-accessible predictors:\n\nnumeric variables based on weather and topography.\nnominal variables based on classifications from other governmental orgs.\n\n\n\n\n\n\nCredit: https://www.svgrepo.com/svg/67614/forest\n\n\n\nThose nominal variables are classifications similar to “forested” but from other agencies. e.g. land_type is from the European Space Agency, and is a remotely-sensed 3-class distribution based on predictions for how the land is used." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#why-not-t-sne", - "href": "slides/advanced-04-feature-engineering-part-one.html#why-not-t-sne", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Why not t-SNE?", - "text": "Why not t-SNE?\nOne of the main requirements for a feature engineering method is that you can reapply the trained transformation done on the training data set to the testing data set.\nThis is not possible with t-SNE as it is an iterative method that shifts observations in the lower-dimensional space based on their distances to points in the higher-dimensional space.\nIt doesn’t create a mapping that can be reused.\n\nVisualizing Data using t-SNE, Laurens van der Maaten and Geoffrey Hinton, Journal of Machine Learning Research, 2008." + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#checklist-for-predictors", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#checklist-for-predictors", + "title": "2 - Your data budget", + "section": "Checklist for predictors", + "text": "Checklist for predictors\n\nIs it ethical to use this variable? (Or even legal?)\nWill this variable be available at prediction time?\nDoes this variable contribute to explainability?\n\n\n\nre: ethics – what issues might arise from releasing the true lat and lon? In reality, these lat and lon are slightly jittered to help ensure trust with landowners who allow surveyers to come take measurements." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca", - "href": "slides/advanced-04-feature-engineering-part-one.html#pca", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "PCA", - "text": "PCA\nPrincipal Compoment Analysis is a linear combination of the original data such that most of the variation is captured in the first variable, then the second, then the third, and so on.\n\nFEAZ, FES" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington-2", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-on-forests-in-washington-2", + "title": "2 - Your data budget", + "section": "Data on forests in Washington", + "text": "Data on forests in Washington\n\nlibrary(tidymodels)\nlibrary(forested)\n\nforested\n#> # A tibble: 7,107 × 19\n#> forested year elevation eastness northness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 Yes 2005 881 90 43 63 Tree 0.04\n#> 2 Yes 2005 113 -25 96 30 Tree 6.4 \n#> 3 No 2005 164 -84 53 13 Tree 6.06\n#> 4 Yes 2005 299 93 34 6 No tree 4.43\n#> 5 Yes 2005 806 47 -88 35 Tree 1.06\n#> 6 Yes 2005 736 -27 -96 53 Tree 1.35\n#> 7 Yes 2005 636 -48 87 3 No tree 1.42\n#> 8 Yes 2005 224 -65 -75 9 Tree 6.39\n#> 9 Yes 2005 52 -62 78 42 Tree 6.5 \n#> 10 Yes 2005 2240 -67 -74 99 No tree -5.63\n#> # ℹ 7,097 more rows\n#> # ℹ 11 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm", - "href": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "PCA Algorithm", - "text": "PCA Algorithm\nThe first principal component of a set of features \\(X_1, X_2, ..., X_p\\) is the normalized linear combination of the features.\n\\[\nZ_1 = \\phi_{11} X_1 + \\phi_{21} X_2 + ... + \\phi_{p1}X_p\n\\]\nthat has the largest variance under the constraint that \\(\\sum_{j=1}^p \\phi_{j1}^2 = 1\\).\nwe refer to \\(\\phi_{11}, ..., \\phi_{p1}\\) as the loadings of the first principal component.\nAnd think of them as the loading vector \\(\\phi_1\\)." + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\nFor machine learning, we typically split data into training and test sets:\n\n\nThe training set is used to estimate model parameters.\nThe test set is used to find an independent assessment of model performance.\n\n\n\nDo not 🚫 use the test set during training." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "PCA Algorithm", - "text": "PCA Algorithm\nsince we have \\(z_{i1} = \\phi_{11} x_{i1} + \\phi_{21} x_{i2} + ... + \\phi_{p1}x_{ip}\\), then we can write\n\\[\n\\underset{\\phi_{11}, ..., \\phi_{p1}}{\\text{maximize}} \\left\\{ \\dfrac{1}{n} \\sum^n_{i=j} z_{i1} ^2 \\right\\} \\quad \\text{subject to} \\quad \\sum_{j=1}^p \\phi_{j1}^2 = 1\n\\]\nWe are, in essence, maximizing the sample variance of the \\(n\\) values of \\(z_{i1}\\).\nWe refer to \\(z_{11}, ..., z_{n1}\\) as the scores of the first principal component." + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-1", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-1", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-2", - "href": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-2", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "PCA Algorithm", - "text": "PCA Algorithm\nLuckily, this can be solved using techniques from Linear Algebra, more specifically, it can be solved using an eigen decomposition.\nOne of the main strengths of PCA is that you don’t need to use optimization to get the results without approximations." + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-2", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-2", + "title": "2 - Your data budget", + "section": "Data splitting and spending", + "text": "Data splitting and spending\n\nSpending too much data in training prevents us from computing a good assessment of predictive performance.\n\n\n\nSpending too much data in testing prevents us from computing a good estimate of model parameters." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-3", - "href": "slides/advanced-04-feature-engineering-part-one.html#pca-algorithm-3", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "PCA Algorithm", - "text": "PCA Algorithm\nOnce the first principal component is calculated, we can calculate the second principal component.\nWe find the second principal component \\(Z_2\\) as a linear combination of \\(X_1, ..., X_p\\) that has the maximal variance out of the linear combinations that are uncorrelated with \\(Z_1\\)\nthis is the same as saying that \\(\\phi_2\\) should be orthogonal to the direction \\(\\phi_1\\)" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nWhen is a good time to split your data?\n\n\n\n−+\n03:00" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#how-is-that-a-dimensionality-reduction-method", - "href": "slides/advanced-04-feature-engineering-part-one.html#how-is-that-a-dimensionality-reduction-method", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "How is that a dimensionality reduction method?", - "text": "How is that a dimensionality reduction method?\nBy itself, it isn’t, as it rotates all the features in the feature space.\nIt becomes a dimensionality reduction method if we only calculate some of the principal components.\nThis is typically done by retaining a specific number of components or as a threshold on the variance explained." + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-initial-split", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-initial-split", + "title": "2 - Your data budget", + "section": "The initial split ", + "text": "The initial split \n\nset.seed(123)\nforested_split <- initial_split(forested)\nforested_split\n#> <Training/Testing/Total>\n#> <5330/1777/7107>\n\n\nHow much data in training vs testing? This function uses a good default, but this depends on your specific goal/data" + }, + { + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#what-is-set.seed", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#what-is-set.seed", + "title": "2 - Your data budget", + "section": "What is set.seed()?", + "text": "What is set.seed()?\nTo create that split of the data, R generates “pseudo-random” numbers: while they are made to behave like random numbers, their generation is deterministic given a “seed”.\nThis allows us to reproduce results by setting that seed.\nWhich seed you pick doesn’t matter, as long as you don’t try a bunch of seeds and pick the one that gives you the best performance." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#different-percent-variance-plots", - "href": "slides/advanced-04-feature-engineering-part-one.html#different-percent-variance-plots", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "4 different percent variance plots", - "text": "4 different percent variance plots" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#accessing-the-data", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#accessing-the-data", + "title": "2 - Your data budget", + "section": "Accessing the data ", + "text": "Accessing the data \n\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#applying-pca-with-recipes", - "href": "slides/advanced-04-feature-engineering-part-one.html#applying-pca-with-recipes", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Applying PCA with recipes ", - "text": "Applying PCA with recipes \nEither use the num_comp argument.\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca(all_numeric_predictors(), num_comp = 5)\n\n\nor using the threshold argument\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca(all_numeric_predictors(), threshold = 0.8)" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-training-set", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-training-set", + "title": "2 - Your data budget", + "section": "The training set", + "text": "The training set\n\nforested_train\n#> # A tibble: 5,330 × 19\n#> forested year elevation eastness northness roughness tree_no_tree dew_temp\n#> <fct> <dbl> <dbl> <dbl> <dbl> <dbl> <fct> <dbl>\n#> 1 No 2016 464 -5 -99 7 No tree 1.71\n#> 2 Yes 2016 166 92 37 7 Tree 6 \n#> 3 No 2016 644 -85 -52 24 No tree 0.67\n#> 4 Yes 2014 1285 4 99 79 Tree 1.91\n#> 5 Yes 2013 822 87 48 68 Tree 1.95\n#> 6 Yes 2017 3 6 -99 5 Tree 7.93\n#> 7 Yes 2014 2041 -95 28 49 Tree -4.22\n#> 8 Yes 2015 1009 -8 99 72 Tree 1.72\n#> 9 No 2017 436 -98 19 10 No tree 1.8 \n#> 10 No 2018 775 63 76 103 No tree 0.62\n#> # ℹ 5,320 more rows\n#> # ℹ 11 more variables: precip_annual <dbl>, temp_annual_mean <dbl>,\n#> # temp_annual_min <dbl>, temp_annual_max <dbl>, temp_january_min <dbl>,\n#> # vapor_min <dbl>, vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>,\n#> # land_type <fct>" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#your-turn-2", - "href": "slides/advanced-04-feature-engineering-part-one.html#your-turn-2", - "title": "4 - Feature engineering: dummies and embeddings", + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-test-set", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-test-set", + "title": "2 - Your data budget", + "section": "The test set ", + "text": "The test set \n🙈\n\nThere are 1777 rows and 19 columns in the test set." + }, + { + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn-1", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn-1", + "title": "2 - Your data budget", "section": "Your turn", - "text": "Your turn\n\nApply PCA using step_pca() to leaf_data data set.\nExperiment with different values of num_comp and or threshold.\n\n\n\n−+\n05:00" + "text": "Your turn\n\nSplit your data so 20% is held out for the test set.\nTry out different values in set.seed() to see how the results change.\n\n\n\n−+\n05:00" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#pca-pros-and-cons", - "href": "slides/advanced-04-feature-engineering-part-one.html#pca-pros-and-cons", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "PCA Pros and Cons", - "text": "PCA Pros and Cons\n\n\nPros\n\nFast\nReliable\nExact results (up to sign changes)\n\n\nCons\n\nComputational time is linear in the number of columns\nCan be quite hard to interpret" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-3", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#data-splitting-and-spending-3", + "title": "2 - Your data budget", + "section": "Data splitting and spending ", + "text": "Data splitting and spending \n\nset.seed(123)\nforested_split <- initial_split(forested, prop = 0.8)\nforested_train <- training(forested_split)\nforested_test <- testing(forested_split)\n\nnrow(forested_train)\n#> [1] 5685\nnrow(forested_test)\n#> [1] 1422" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#truncated-pca", - "href": "slides/advanced-04-feature-engineering-part-one.html#truncated-pca", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Truncated PCA", - "text": "Truncated PCA\n\nComputational time is linear in the number of columns\n\nBy default, step_pca() calculates all the loading vectors. And then subset them down to what you need.\nInstead, we can use a different implementation that only calculates what you need. This is what we call truncated PCA.\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn-2", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#your-turn-2", + "title": "2 - Your data budget", + "section": "Your turn", + "text": "Your turn\n\nExplore the forested_train data on your own!\n\nWhat’s the distribution of the outcome, forested?\nWhat’s the distribution of numeric variables like precip_annual?\nHow does the distribution of forested differ across the categorical variables?\n\n\n\n\n−+\n08:00\n\n\n\n\nMake a plot or summary and then share with neighbor" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#truncated-pca-with-recipes", - "href": "slides/advanced-04-feature-engineering-part-one.html#truncated-pca-with-recipes", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Truncated PCA with recipes ", - "text": "Truncated PCA with recipes \nCan only be done using num_comp\n\nlibrary(embed)\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_truncated(all_numeric_predictors(), num_comp = 5)" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-1", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-1", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train %>% \n ggplot(aes(x = forested)) +\n geom_bar()" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca", - "href": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Sparse PCA", - "text": "Sparse PCA\n\nCan be quite hard to interpret\n\nEvery component is a linear combination of all predictors.\n\\[\n\\begin{alignat}{4}\nPC1 &= cyl \\cdot -0.021 &&+ disp \\cdot -0.85 &&+ hp \\cdot -0.52\\\\\nPC2 &= cyl \\cdot 0.013 &&+ disp \\cdot -0.52 &&+ hp \\cdot 0.85\\\\\nPC3 &= cyl \\cdot -0.12 &&+ disp \\cdot 0.016 &&+ hp \\cdot 0.081\\\\\nPC4 &= cyl \\cdot -0.22 &&+ disp \\cdot -0.0061 &&+ hp \\cdot 0.033\\\\\nPC5 &= cyl \\cdot 0.73 &&+ disp \\cdot -0.014 &&+ hp \\cdot 0.0016\n\\end{alignat}\n\\]\nIf we could force some of the loadings to be 0, it would reduce things a lot.\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-2", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-2", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train %>% \n ggplot(aes(x = forested, fill = tree_no_tree)) +\n geom_bar()" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes", - "href": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Sparse PCA with recipes ", - "text": "Sparse PCA with recipes \nCan only be done using num_comp.\nThe predictor_prop argument is used to determine how many zeroes in the loadings.\n\nlibrary(embed)\n\nrec <- recipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_sparse(all_numeric_predictors(), num_comp = 5, predictor_prop = 0.8)" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-3", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-3", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train %>% \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"identity\", alpha = .7)" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes-1", - "href": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes-1", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Sparse PCA with recipes ", - "text": "Sparse PCA with recipes \npredictor_prop = 0.8\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_sparse(all_numeric_predictors(), num_comp = 4, predictor_prop = 0.8) |>\n prep() |>\n tidy(number = 2) |>\n pivot_wider(names_from = component, values_from = value) |>\n select(-id)\n#> # A tibble: 10 × 5\n#> terms PC1 PC2 PC3 PC4\n#> <chr> <dbl> <dbl> <dbl> <dbl>\n#> 1 cyl -0.503 0 0.0755 0 \n#> 2 disp -0.504 0 0 0.227 \n#> 3 hp -0.377 -0.228 -0.0985 0 \n#> 4 drat 0.267 -0.262 0 0.911 \n#> 5 wt -0.420 0.105 -0.383 0.236 \n#> 6 qsec 0 0.473 -0.425 0.0966 \n#> 7 vs 0.313 0.203 -0.429 -0.0774 \n#> 8 am 0.0687 -0.440 0.191 -0.00802\n#> 9 gear 0 -0.482 -0.237 -0.208 \n#> 10 carb -0.00616 -0.422 -0.617 -0.0654" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-4", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-4", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train %>% \n ggplot(aes(x = precip_annual, fill = forested, group = forested)) +\n geom_histogram(position = \"fill\")" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes-2", - "href": "slides/advanced-04-feature-engineering-part-one.html#sparse-pca-with-recipes-2", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Sparse PCA with recipes ", - "text": "Sparse PCA with recipes \npredictor_prop = 0.2\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_normalize(all_predictors()) |>\n step_pca_sparse(all_numeric_predictors(), num_comp = 4, predictor_prop = 0.2) |>\n prep() |>\n tidy(number = 2) |>\n pivot_wider(names_from = component, values_from = value) |>\n select(-id)\n#> # A tibble: 10 × 5\n#> terms PC1 PC2 PC3 PC4\n#> <chr> <dbl> <dbl> <dbl> <dbl>\n#> 1 cyl 0.712 0 0 0 \n#> 2 disp 0.702 0 0 0 \n#> 3 hp 0 0 0 0 \n#> 4 drat 0 0 0 0.978\n#> 5 wt 0 0 -0.278 0 \n#> 6 qsec 0 0.666 0 0 \n#> 7 vs 0 0 0 0 \n#> 8 am 0 0 0 0.211\n#> 9 gear 0 -0.746 0 0 \n#> 10 carb 0 0 -0.961 0" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-5", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#section-5", + "title": "2 - Your data budget", + "section": "", + "text": "forested_train %>% \n ggplot(aes(x = lon, y = lat, col = forested)) +\n geom_point()" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#nnmf", - "href": "slides/advanced-04-feature-engineering-part-one.html#nnmf", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "NNMF", - "text": "NNMF\nNon-Negative Matrix Factorization is conceptually similar to PCA, but it has different objectives.\nPCA aims to generate uncorrelated components that maximize the variances. One component at a time.\nNNMF, on the other hand, simultaneously optimizes all the components under the constraint that all the loadings are non-negative. While the data is also non-negative.\n\nFEAZ FES" + "objectID": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-whole-game---status-update", + "href": "archive/2024-08-posit-conf/intro-02-data-budget.html#the-whole-game---status-update", + "title": "2 - Your data budget", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#nnmf-restriction", - "href": "slides/advanced-04-feature-engineering-part-one.html#nnmf-restriction", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "NNMF restriction", - "text": "NNMF restriction\nThe data has to be non-negative, i.e., 0 or higher.\nThis might feel like a pretty big restriction. And that is not wrong. But a lot of data sets end up being naturally non-negative.\nOr could at least be turned into non-negative ones with transformations. As long as you don’t scale them below 0.\nIt makes for much easier interpretations as the loadings don’t cancel each other out like they do for PCA." + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#looking-at-predictions", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#looking-at-predictions", + "title": "4 - Evaluating models", + "section": "Looking at predictions", + "text": "Looking at predictions\n\naugment(forested_fit, new_data = forested_train)\n#> # A tibble: 5,685 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness northness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 No 0.0114 0.989 No 2016 464 -5 -99\n#> 2 Yes 0.636 0.364 Yes 2016 166 92 37\n#> 3 No 0.0114 0.989 No 2016 644 -85 -52\n#> 4 Yes 0.977 0.0226 Yes 2014 1285 4 99\n#> 5 Yes 0.977 0.0226 Yes 2013 822 87 48\n#> 6 Yes 0.808 0.192 Yes 2017 3 6 -99\n#> 7 Yes 0.977 0.0226 Yes 2014 2041 -95 28\n#> 8 Yes 0.977 0.0226 Yes 2015 1009 -8 99\n#> 9 No 0.0114 0.989 No 2017 436 -98 19\n#> 10 No 0.0114 0.989 No 2018 775 63 76\n#> # ℹ 5,675 more rows\n#> # ℹ 14 more variables: roughness <dbl>, tree_no_tree <fct>, dew_temp <dbl>,\n#> # precip_annual <dbl>, temp_annual_mean <dbl>, temp_annual_min <dbl>,\n#> # temp_annual_max <dbl>, temp_january_min <dbl>, vapor_min <dbl>,\n#> # vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#nnmf-pros-and-cons", - "href": "slides/advanced-04-feature-engineering-part-one.html#nnmf-pros-and-cons", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "NNMF Pros and Cons", - "text": "NNMF Pros and Cons\n\n\n\nMore interpretable results\nPulls out better structures\n\n\n\nData must be non-negative\nComputationally expensive\nTraining depends on the seed" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#nnmf-with-recipes", - "href": "slides/advanced-04-feature-engineering-part-one.html#nnmf-with-recipes", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "NNMF with recipes ", - "text": "NNMF with recipes \n\nset.seed(1234)\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_nnmf_sparse(all_numeric_predictors(), num_comp = 2) |>\n prep() |>\n tidy(number = 1) |>\n pivot_wider(names_from = component, values_from = value) |>\n select(-id)\n#> # A tibble: 10 × 3\n#> terms NNMF1 NNMF2\n#> <chr> <dbl> <dbl>\n#> 1 am 0 0.00631\n#> 2 carb 0.00428 0.0249 \n#> 3 cyl 0.0134 0.0177 \n#> 4 disp 0.639 0 \n#> 5 drat 0.00545 0.0188 \n#> 6 gear 0.00509 0.0240 \n#> 7 hp 0.294 0.840 \n#> 8 qsec 0.0316 0.0606 \n#> 9 vs 0.000293 0.00247\n#> 10 wt 0.00742 0.00519" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix-1", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) %>%\n conf_mat(truth = forested, estimate = .pred_class)\n#> Truth\n#> Prediction Yes No\n#> Yes 2991 176\n#> No 144 2374" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap", - "href": "slides/advanced-04-feature-engineering-part-one.html#umap", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "UMAP", - "text": "UMAP\nUniform Manifold Approximation and Projection is another method that takes high-dimensional data and transforms it into a lower-dimensional space.\nRuns relatively fast and is popular in visualizations." + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix-2", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#confusion-matrix-2", + "title": "4 - Evaluating models", + "section": "Confusion matrix ", + "text": "Confusion matrix \n\naugment(forested_fit, new_data = forested_train) %>%\n conf_mat(truth = forested, estimate = .pred_class) %>%\n autoplot(type = \"heatmap\")" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap-algorithm", - "href": "slides/advanced-04-feature-engineering-part-one.html#umap-algorithm", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "UMAP Algorithm", - "text": "UMAP Algorithm\nRough algorithm\n\nUse spectral embedding to embed points in a low-dimensional space\nCalculate similarity scores between points based on the original data set\nRandomly samples a pair of points based on their similarity scores\nFlips a coin to decide which of the pair of points to give to the other one\nRandomly picks a non-neighbor point to move away from\nMoves the selected point towards its neighbor and away from its non-neighbor\nRepeat 3-6" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) %>%\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944\n\n\n\n\n\nThere used to be a slide here calling out the pitfalls of accuracy when classes are imbalanced." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap-parameters", - "href": "slides/advanced-04-feature-engineering-part-one.html#umap-parameters", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "UMAP parameters", - "text": "UMAP parameters\n\n\n\nn_neighbors\n\n\nDetermines how many points are considered neighbors. A point counts as its own neighbor. Lower values lead to a local view.\n\n\n\n\nmin_dist\n\n\nDetermines how close points are allowed to be to each other in the low-dimensional space.\n\n\n\n\nmetric\n\n\nHow distances are calculated in the input data: euclidean, manhattan, jaccard, etc." + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-1", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) %>%\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.954" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap-hesitancy", - "href": "slides/advanced-04-feature-engineering-part-one.html#umap-hesitancy", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "UMAP hesitancy", - "text": "UMAP hesitancy\nDue to the flexibility of how this method works, it is almost always possible to generate graphs that appear to have insights in them." + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-2", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-2", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \n\n\n\naugment(forested_fit, new_data = forested_train) %>%\n sensitivity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 sensitivity binary 0.954\n\n\n\naugment(forested_fit, new_data = forested_train) %>%\n specificity(truth = forested, estimate = .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 specificity binary 0.931" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#which-was-created-with-random-data", - "href": "slides/advanced-04-feature-engineering-part-one.html#which-was-created-with-random-data", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Which was created with random data?", - "text": "Which was created with random data?" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-3", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-3", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \nWe can use metric_set() to combine multiple calculations into one\n\nforested_metrics <- metric_set(accuracy, specificity, sensitivity)\n\naugment(forested_fit, new_data = forested_train) %>%\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 3 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944\n#> 2 specificity binary 0.931\n#> 3 sensitivity binary 0.954" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#all-of-them-different-n_neighbors", - "href": "slides/advanced-04-feature-engineering-part-one.html#all-of-them-different-n_neighbors", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "All of them! Different n_neighbors", - "text": "All of them! Different n_neighbors" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-4", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#metrics-for-model-performance-4", + "title": "4 - Evaluating models", + "section": "Metrics for model performance ", + "text": "Metrics for model performance \nMetrics and metric sets work with grouped data frames!\n\naugment(forested_fit, new_data = forested_train) %>%\n group_by(tree_no_tree) %>%\n accuracy(truth = forested, estimate = .pred_class)\n#> # A tibble: 2 × 4\n#> tree_no_tree .metric .estimator .estimate\n#> <fct> <chr> <chr> <dbl>\n#> 1 Tree accuracy binary 0.946\n#> 2 No tree accuracy binary 0.941" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#umap-with-recipes", - "href": "slides/advanced-04-feature-engineering-part-one.html#umap-with-recipes", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "umap with recipes ", - "text": "umap with recipes \n\nlibrary(embed)\nset.seed(1234)\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_umap(all_numeric_predictors()) |>\n prep() |>\n bake(NULL)\n#> # A tibble: 32 × 3\n#> mpg UMAP1 UMAP2\n#> <dbl> <dbl> <dbl>\n#> 1 21 1.50 2.26\n#> 2 21 1.26 2.02\n#> 3 22.8 2.17 4.19\n#> 4 21.4 -1.38 -1.67\n#> 5 18.7 -2.67 -2.66\n#> 6 18.1 -0.759 -1.57\n#> 7 14.3 -3.19 -3.37\n#> 8 24.4 1.31 3.91\n#> 9 22.8 2.06 2.46\n#> 10 19.2 0.876 2.31\n#> # ℹ 22 more rows" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nApply the forested_metrics metric set to augment()\noutput grouped by tree_no_tree.\nDo any metrics differ substantially between groups?\n\n\n\n−+\n05:00\n\n\n\n\nThe specificity for \"Tree\" is a good bit lower than it is for \"No tree\".\nSpecificity is the proportion of negatives that are correctly identified as negatives. “Negative” is the non-event level of the outcome, i.e. “non-forested.” So, when this index classifies the plot as having a tree, the model does not do well at correctly identifying the plot as non-forested when it is indeed non-forested." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#isomap", - "href": "slides/advanced-04-feature-engineering-part-one.html#isomap", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Isomap", - "text": "Isomap\nIsometric mapping is a non-linear dimensionality reduction method.\nThis is another method that uses distances between points to produce graphs of neighboring points. Where this method is different than other methods is that it uses geodesic distances as opposed to straight-line distances.\nThe geodesic distance is the sum of edge weights along the shortest path between two points.\nThe eigenvectors of the deodesic distance metric are then used to represent the new coordinates." + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#two-class-data", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#two-class-data", + "title": "4 - Evaluating models", + "section": "Two class data", + "text": "Two class data\nThese metrics assume that we know the threshold for converting “soft” probability predictions into “hard” class predictions.\n\nIs a 50% threshold good?\nWhat happens if we say that we need to be 80% sure to declare an event?\n\nsensitivity ⬇️, specificity ⬆️\n\n\n\nWhat happens for a 20% threshold?\n\nsensitivity ⬆️, specificity ⬇️" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#isomap-algorithm", - "href": "slides/advanced-04-feature-engineering-part-one.html#isomap-algorithm", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Isomap Algorithm", - "text": "Isomap Algorithm\nA very high-level description of the Isomap algorithm is given below.\n\nFind the neighbors for each point\nConstruct the neighborhood graph, using Euclidean distance as edge length\nCalculate the shortest path between each pair of points\nUse Multidimensional scaling to compute a lower-dimensional embedding" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#varying-the-threshold", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#varying-the-threshold", + "title": "4 - Evaluating models", + "section": "Varying the threshold", + "text": "Varying the threshold" }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#isomap-pros-and-cons", - "href": "slides/advanced-04-feature-engineering-part-one.html#isomap-pros-and-cons", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "Isomap Pros and Cons", - "text": "Isomap Pros and Cons\n\n\nPros\n\nCaptures non-linear effects\nCaptures long-range structure, not just local structure\nNo parameters to set other than neighbors\n\n\nCons\n\nComputationally expensive\nAssumes a single connected manifold" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves", + "title": "4 - Evaluating models", + "section": "ROC curves", + "text": "ROC curves\n\n\nFor an ROC (receiver operator characteristic) curve, we plot\n\nthe false positive rate (1 - specificity) on the x-axis\nthe true positive rate (sensitivity) on the y-axis\n\nwith sensitivity and specificity calculated at all possible thresholds.\n\n\n\n\n\n\n\n\n\n\n\n\nROC curves are insensitive to class imbalance." }, { - "objectID": "slides/advanced-04-feature-engineering-part-one.html#isomap-with-recipes", - "href": "slides/advanced-04-feature-engineering-part-one.html#isomap-with-recipes", - "title": "4 - Feature engineering: dummies and embeddings", - "section": "isomap with recipes ", - "text": "isomap with recipes \n\nlibrary(embed)\nset.seed(1234)\n\nrecipe(mpg ~ ., data = mtcars) |>\n step_isomap(all_numeric_predictors(), neighbors = 10) |>\n prep() |>\n bake(NULL)\n#> # A tibble: 32 × 6\n#> mpg Isomap1 Isomap2 Isomap3 Isomap4 Isomap5\n#> <dbl> <dbl> <dbl> <dbl> <dbl> <dbl>\n#> 1 21 -98.5 35.6 35.3 137. 105. \n#> 2 21 -98.5 35.6 35.1 137. 104. \n#> 3 22.8 -161. 48.1 20.4 160. 129. \n#> 4 21.4 8.55 14.6 37.2 86.8 125. \n#> 5 18.7 136. 28.5 35.7 -30.8 102. \n#> 6 18.1 -27.7 21.7 38.3 104. 116. \n#> 7 14.3 207. -16.5 34.6 -55.5 158. \n#> 8 24.4 -147. 44.2 23.7 182. 88.4\n#> 9 22.8 -125. 41.1 18.1 151. 118. \n#> 10 19.2 -91.6 33.7 48.6 135. 101. \n#> # ℹ 22 more rows" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves-1", + "title": "4 - Evaluating models", + "section": "ROC curves", + "text": "ROC curves\n\n\nWe can use the area under the ROC curve as a classification metric:\n\nROC AUC = 1 💯\nROC AUC = 1/2 😢" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#getting-set-up", - "href": "slides/advanced-05-feature-engineering-part-two.html#getting-set-up", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Getting set up ", - "text": "Getting set up \n\nlibrary(tidymodels)\nlibrary(embed)\nlibrary(extrasteps)\n\ntidymodels_prefer()\ntheme_set(theme_bw())\noptions(pillar.advice = FALSE, pillar.min_title_chars = Inf)" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves-2", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curves-2", + "title": "4 - Evaluating models", + "section": "ROC curves ", + "text": "ROC curves \n\n# Assumes _first_ factor level is event; there are options to change that\naugment(forested_fit, new_data = forested_train) %>% \n roc_curve(truth = forested, .pred_Yes) %>%\n slice(1, 20, 50)\n#> # A tibble: 3 × 3\n#> .threshold specificity sensitivity\n#> <dbl> <dbl> <dbl>\n#> 1 -Inf 0 1 \n#> 2 0.235 0.885 0.972\n#> 3 0.909 0.969 0.826\n\naugment(forested_fit, new_data = forested_train) %>% \n roc_auc(truth = forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 roc_auc binary 0.975" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#hotel-rates-data-set", - "href": "slides/advanced-05-feature-engineering-part-two.html#hotel-rates-data-set", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Hotel rates data set", - "text": "Hotel rates data set\nRegression data set for predicting the average daily rate for a room, for “Resort Hotel”. The agent and company use random names.\n\nglimpse(hotel_rates)\n#> Rows: 15,402\n#> Columns: 28\n#> $ avg_price_per_room <dbl> 110.00, 74.00, 81.90, 81.00, 112.20, 90…\n#> $ lead_time <dbl> 241, 273, 248, 236, 243, 267, 94, 10, 1…\n#> $ stays_in_weekend_nights <dbl> 0, 2, 2, 2, 4, 2, 4, 0, 0, 0, 0, 0, 0, …\n#> $ stays_in_week_nights <dbl> 1, 5, 5, 5, 10, 5, 7, 1, 1, 1, 1, 1, 1,…\n#> $ adults <dbl> 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, …\n#> $ children <dbl> 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, …\n#> $ babies <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ meal <fct> bed_and_breakfast, bed_and_breakfast, b…\n#> $ country <fct> prt, aus, gbr, prt, gbr, null, prt, esp…\n#> $ market_segment <fct> online_travel_agent, offline_travel_age…\n#> $ distribution_channel <fct> ta_to, ta_to, ta_to, ta_to, ta_to, ta_t…\n#> $ is_repeated_guest <dbl> 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, …\n#> $ previous_cancellations <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ previous_bookings_not_canceled <dbl> 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, …\n#> $ reserved_room_type <fct> a, a, a, a, a, a, f, e, h, a, a, g, a, …\n#> $ assigned_room_type <fct> c, a, c, a, a, a, f, f, h, e, e, g, e, …\n#> $ booking_changes <dbl> 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, …\n#> $ agent <fct> devin_rivera_borrego, lia_nauth, jawhar…\n#> $ company <fct> not_applicable, not_applicable, not_app…\n#> $ days_in_waiting_list <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ customer_type <fct> transient, transient_party, transient, …\n#> $ required_car_parking_spaces <dbl> 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, …\n#> $ total_of_special_requests <dbl> 1, 0, 0, 2, 0, 0, 1, 1, 0, 2, 2, 0, 2, …\n#> $ arrival_date <date> 2016-07-02, 2016-07-02, 2016-07-02, 20…\n#> $ arrival_date_num <dbl> 2016.5, 2016.5, 2016.5, 2016.5, 2016.5,…\n#> $ near_christmas <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ near_new_years <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …\n#> $ historical_adr <dbl> 104.9811, 104.9811, 104.9811, 104.9811,…\n\n\nAntonio, N., de Almeida, A., and Nunes, L. (2019). Hotel booking demand datasets. Data in Brief, 22, 41-49." + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curve-plot", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#roc-curve-plot", + "title": "4 - Evaluating models", + "section": "ROC curve plot ", + "text": "ROC curve plot \n\n\naugment(forested_fit, \n new_data = forested_train) %>% \n roc_curve(truth = forested, \n .pred_Yes) %>%\n autoplot()" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#hotel-data-splitting", - "href": "slides/advanced-05-feature-engineering-part-two.html#hotel-data-splitting", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Hotel data splitting ", - "text": "Hotel data splitting \nGenerally, you should always do data splitting. We are doing it here explicitly because some artifacts of splitting data become useful later on.\n\nset.seed(1234)\nhotel_split <- initial_split(hotel_rates)\nhotel_train <- training(hotel_split)\nhotel_test <- testing(hotel_split)" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-1", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nCompute and plot an ROC curve for your current model.\nWhat data are being used for this ROC curve plot?\n\n\n\n−+\n05:00" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn", - "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Your turn", - "text": "Your turn\n\nLoad and explore the hotel_train data\nComes loaded with the modeldata package\n\n\n\n−+\n05:00" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#brier-score", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#brier-score", + "title": "4 - Evaluating models", + "section": "Brier score", + "text": "Brier score\nWhat if we don’t turn predicted probabilities into class predictions?\n\nThe Brier score is analogous to the mean squared error in regression models:\n\\[\nBrier_{class} = \\frac{1}{N}\\sum_{i=1}^N\\sum_{k=1}^C (y_{ik} - \\hat{p}_{ik})^2\n\\]" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#nonlinear-predictors", - "href": "slides/advanced-05-feature-engineering-part-two.html#nonlinear-predictors", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Nonlinear predictors", - "text": "Nonlinear predictors" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#brier-score-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#brier-score-1", + "title": "4 - Evaluating models", + "section": "Brier score", + "text": "Brier score\n\naugment(forested_fit, new_data = forested_train) %>% \n brier_class(truth = forested, .pred_Yes) \n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0469\n\n\nSmaller values are better, for binary classification the “bad model threshold” is about 0.25." }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#splines-1", - "href": "slides/advanced-05-feature-engineering-part-two.html#splines-1", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Splines", - "text": "Splines\nIt is a way to transform a single numeric predictor into multiple numeric predictors, with the hope that the new numeric predictors are more linearly related to the outcome.\nMostly needed with linear models, but it should rarely hurt to use it.\nIf you’ve ever used geom_smooth() you have seen splines in action.\n\nFEAZ FES" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#separation-vs-calibration", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#separation-vs-calibration", + "title": "4 - Evaluating models", + "section": "Separation vs calibration", + "text": "Separation vs calibration\n\n\nThe ROC captures separation.\n\n\n\n\n\n\n\n\n\n\nThe Brier score captures calibration.\n\n\n\n\n\n\n\n\n\n\n\n\nGood separation: the densities don’t overlap.\nGood calibration: the calibration line follows the diagonal.\n\nCalibration plot: We bin observations according to predicted probability. In the bin for 20%-30% predicted prob, we should see an event rate of ~25% if the model is well-calibrated." }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#splines-explained", - "href": "slides/advanced-05-feature-engineering-part-two.html#splines-explained", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Splines explained", - "text": "Splines explained\nA spline is a piecewise polynomial function.\nWe have 2 main parameters to worry about. Number of knots and the polynomial degree.\nThe domain of the predictor is split into k regions, with a knot between each, and a polynomial function is fit within each region, under the constraint that they touch each other at the knot." + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-1", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting", + "text": "Dangers of overfitting" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-1-degree-1", - "href": "slides/advanced-05-feature-engineering-part-two.html#knots-1-degree-1", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "knots: 1, degree: 1", - "text": "knots: 1, degree: 1" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-2", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-2", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️", + "text": "Dangers of overfitting ⚠️" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-2-degree-1", - "href": "slides/advanced-05-feature-engineering-part-two.html#knots-2-degree-1", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "knots: 2, degree: 1", - "text": "knots: 2, degree: 1" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-3", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-3", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\nforested_fit %>%\n augment(forested_train)\n#> # A tibble: 5,685 × 22\n#> .pred_class .pred_Yes .pred_No forested year elevation eastness northness\n#> <fct> <dbl> <dbl> <fct> <dbl> <dbl> <dbl> <dbl>\n#> 1 No 0.0114 0.989 No 2016 464 -5 -99\n#> 2 Yes 0.636 0.364 Yes 2016 166 92 37\n#> 3 No 0.0114 0.989 No 2016 644 -85 -52\n#> 4 Yes 0.977 0.0226 Yes 2014 1285 4 99\n#> 5 Yes 0.977 0.0226 Yes 2013 822 87 48\n#> 6 Yes 0.808 0.192 Yes 2017 3 6 -99\n#> 7 Yes 0.977 0.0226 Yes 2014 2041 -95 28\n#> 8 Yes 0.977 0.0226 Yes 2015 1009 -8 99\n#> 9 No 0.0114 0.989 No 2017 436 -98 19\n#> 10 No 0.0114 0.989 No 2018 775 63 76\n#> # ℹ 5,675 more rows\n#> # ℹ 14 more variables: roughness <dbl>, tree_no_tree <fct>, dew_temp <dbl>,\n#> # precip_annual <dbl>, temp_annual_mean <dbl>, temp_annual_min <dbl>,\n#> # temp_annual_max <dbl>, temp_january_min <dbl>, vapor_min <dbl>,\n#> # vapor_max <dbl>, canopy_cover <dbl>, lon <dbl>, lat <dbl>, land_type <fct>\n\nWe call this “resubstitution” or “repredicting the training set”" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-1", - "href": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-1", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "knots: 5, degree: 1", - "text": "knots: 5, degree: 1" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-4", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-4", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\nforested_fit %>%\n augment(forested_train) %>%\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944\n\nWe call this a “resubstitution estimate”" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-2", - "href": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-2", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "knots: 5, degree: 2", - "text": "knots: 5, degree: 2" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-5", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-5", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit %>%\n augment(forested_train) %>%\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-3", - "href": "slides/advanced-05-feature-engineering-part-two.html#knots-5-degree-3", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "knots: 5, degree: 3", - "text": "knots: 5, degree: 3" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-6", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-6", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit %>%\n augment(forested_train) %>%\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.944\n\n\n\nforested_fit %>%\n augment(forested_test) %>%\n accuracy(forested, .pred_class)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 accuracy binary 0.886\n\n\n\n⚠️ Remember that we’re demonstrating overfitting\n\n\n⚠️ Don’t use the test set until the end of your modeling analysis" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#knots-9-degree-3", - "href": "slides/advanced-05-feature-engineering-part-two.html#knots-9-degree-3", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "knots: 9, degree: 3", - "text": "knots: 9, degree: 3" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-2", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-2", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse augment() and a metric function to compute a classification metric like brier_class().\nCompute the metrics for both training and testing data to demonstrate overfitting!\nNotice the evidence of overfitting! ⚠️\n\n\n\n−+\n05:00" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#b-spline-features-visualized---degree-3", - "href": "slides/advanced-05-feature-engineering-part-two.html#b-spline-features-visualized---degree-3", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "B-Spline features visualized - degree: 3", - "text": "B-Spline features visualized - degree: 3" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-7", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#dangers-of-overfitting-7", + "title": "4 - Evaluating models", + "section": "Dangers of overfitting ⚠️ ", + "text": "Dangers of overfitting ⚠️ \n\n\n\nforested_fit %>%\n augment(forested_train) %>%\n brier_class(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0469\n\n\n\nforested_fit %>%\n augment(forested_test) %>%\n brier_class(forested, .pred_Yes)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> <chr> <chr> <dbl>\n#> 1 brier_class binary 0.0888\n\n\n\nWhat if we want to compare more models?\n\n\nAnd/or more model configurations?\n\n\nAnd we want to understand if these are important differences?" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#splines-as-numbers", - "href": "slides/advanced-05-feature-engineering-part-two.html#splines-as-numbers", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Splines as numbers", - "text": "Splines as numbers\n\n\n\n\n\n\n\n\n\n\n\n\n\n\narrival_date_num\nSpline Feature 1\nSpline Feature 2\nSpline Feature 3\nSpline Feature 4\nSpline Feature 5\nSpline Feature 6\n\n\n\n\n2017.619\n0.00\n0.00\n0.00\n0.03\n0.35\n0.62\n\n\n2016.844\n0.15\n0.59\n0.26\n0.00\n0.00\n0.00\n\n\n2016.702\n0.51\n0.40\n0.05\n0.00\n0.00\n0.00\n\n\n2017.077\n0.00\n0.19\n0.67\n0.14\n0.00\n0.00\n\n\n2016.861\n0.13\n0.58\n0.29\n0.00\n0.00\n0.00\n\n\n2017.019\n0.00\n0.30\n0.62\n0.07\n0.00\n0.00\n\n\n2017.123\n0.00\n0.11\n0.66\n0.23\n0.00\n0.00" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#splines-pros-and-cons", - "href": "slides/advanced-05-feature-engineering-part-two.html#splines-pros-and-cons", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Splines pros and cons", - "text": "Splines pros and cons\n\n\nPros\n\nfast\neasy to use\nsemi-interpretable\n\n\nCons\n\nadds more columns\nwill need to select the number of columns\ncan be messy outside the range" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-1", + "title": "4 - Evaluating models", + "section": "Cross-validation", + "text": "Cross-validation" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn-1", - "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn-1", - "title": "5 - Feature engineering: splines, target encoding and dates", + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-3", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-3", + "title": "4 - Evaluating models", "section": "Your turn", - "text": "Your turn\n\nApply B-splines to some variables using step_spline_b()\n\n\n\n−+\n03:00" + "text": "Your turn\n\nIf we use 10 folds, what percent of the training data\n\nends up in analysis\nends up in assessment\n\nfor each fold?\n\n\n\n\n−+\n03:00" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#factors-with-many-categories", - "href": "slides/advanced-05-feature-engineering-part-two.html#factors-with-many-categories", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Factors with many categories", - "text": "Factors with many categories\n\n\n\nhotel_train |>\n count(country)\n#> # A tibble: 88 × 2\n#> country n\n#> <fct> <int>\n#> 1 ago 7\n#> 2 and 1\n#> 3 are 3\n#> 4 arg 14\n#> 5 aus 31\n#> 6 aut 76\n#> 7 aze 2\n#> 8 bel 184\n#> 9 bgr 2\n#> 10 bhs 1\n#> # ℹ 78 more rows\n\n\n\nhotel_train |>\n count(company)\n#> # A tibble: 157 × 2\n#> company n\n#> <fct> <int>\n#> 1 abdou_llc 2\n#> 2 afework_llc 5\n#> 3 alston_pbc 3\n#> 4 battle_llc 23\n#> 5 bennett_and_company 3\n#> 6 berhanu_pbc 53\n#> 7 biggers_llc 4\n#> 8 blasingime_llc 1\n#> 9 boddy_llc 16\n#> 10 boles_pbc 3\n#> # ℹ 147 more rows\n\n\n\nhotel_train |>\n count(agent)\n#> # A tibble: 119 × 2\n#> agent n\n#> <fct> <int>\n#> 1 aaron_marquez 2\n#> 2 alexander_drake 1117\n#> 3 allen_her 1\n#> 4 anas_el_bashir 1\n#> 5 araseli_billy 1\n#> 6 arhab_al_islam 7\n#> 7 audray_tucker 38\n#> 8 bernice_baltierra 35\n#> 9 betzy_rodriguez 66\n#> 10 brayan_guerrero 2\n#> # ℹ 109 more rows" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-2", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-2", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(forested_train) # v = 10 is default\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5116/569]> Fold01\n#> 2 <split [5116/569]> Fold02\n#> 3 <split [5116/569]> Fold03\n#> 4 <split [5116/569]> Fold04\n#> 5 <split [5116/569]> Fold05\n#> 6 <split [5117/568]> Fold06\n#> 7 <split [5117/568]> Fold07\n#> 8 <split [5117/568]> Fold08\n#> 9 <split [5117/568]> Fold09\n#> 10 <split [5117/568]> Fold10" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#how-do-we-handle-them", - "href": "slides/advanced-05-feature-engineering-part-two.html#how-do-we-handle-them", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "How do we handle them?", - "text": "How do we handle them?\n\nWe could:\n\nMake the full set of indicator variables 😳\nLump agents and companies that rarely occur into an “other” group\nUse feature hashing to create a smaller set of indicator variables\nUse target encoding to replace the county, agent, and company columns with the estimated effect of that predictor" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-3", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-3", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWhat is in this?\n\nforested_folds <- vfold_cv(forested_train)\nforested_folds$splits[1:3]\n#> [[1]]\n#> <Analysis/Assess/Total>\n#> <5116/569/5685>\n#> \n#> [[2]]\n#> <Analysis/Assess/Total>\n#> <5116/569/5685>\n#> \n#> [[3]]\n#> <Analysis/Assess/Total>\n#> <5116/569/5685>\n\n\nTalk about a list column, storing non-atomic types in dataframe" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-1", - "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-1", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding", - "text": "Target encoding\nTarget encoding (also called mean encoding, likelihood encoding, impact encoding, or effect encoding) is a supervised trained method that turns a single categorical predictor into a single numeric predictor.\nIt is often used to deal with categorical predictors with many levels, although it works regardless.\nSince it uses the outcome to train it, you need to make sure to use cross-validation to avoid overfitting.\n\nFEAZ FES" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-4", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-4", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \n\nvfold_cv(forested_train, v = 5)\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> <list> <chr>\n#> 1 <split [4548/1137]> Fold1\n#> 2 <split [4548/1137]> Fold2\n#> 3 <split [4548/1137]> Fold3\n#> 4 <split [4548/1137]> Fold4\n#> 5 <split [4548/1137]> Fold5" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-motivation", - "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-motivation", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding motivation", - "text": "Target encoding motivation\nYou have a numeric outcome and a categorical predictor. And you want to transform each value of the categorical predictor into a value that best represents the outcome?\n\n\n\nWe calculate the mean of the outcome within each level of the predictor, and use that as the new value.\n\n\n\n\n\n\nCaution\n\n\nDon’t do just this! We are building up the method one thing at a time. Unregularized target encoding is really prone to overfitting.\n\n\n\n\n\nhotel_train |>\n summarise(\n mean = mean(avg_price_per_room),\n .by = agent\n )\n#> # A tibble: 119 × 2\n#> agent mean\n#> <fct> <dbl>\n#> 1 alexander_drake 144. \n#> 2 kaylae_maxedon 62.5\n#> 3 michael_mcdole 60.9\n#> 4 devin_rivera_borrego 126. \n#> 5 james_richards 78.6\n#> 6 estela_bonilla 41.9\n#> 7 charles_najera 109. \n#> 8 reema_el_tamer 118. \n#> 9 jawhara_al_azad 90.1\n#> 10 not_applicable 84.1\n#> # ℹ 109 more rows" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-5", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#cross-validation-5", + "title": "4 - Evaluating models", + "section": "Cross-validation ", + "text": "Cross-validation \nWe’ll use this setup:\n\nset.seed(123)\nforested_folds <- vfold_cv(forested_train, v = 10)\nforested_folds\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5116/569]> Fold01\n#> 2 <split [5116/569]> Fold02\n#> 3 <split [5116/569]> Fold03\n#> 4 <split [5116/569]> Fold04\n#> 5 <split [5116/569]> Fold05\n#> 6 <split [5117/568]> Fold06\n#> 7 <split [5117/568]> Fold07\n#> 8 <split [5117/568]> Fold08\n#> 9 <split [5117/568]> Fold09\n#> 10 <split [5117/568]> Fold10\n\n\nSet the seed when creating resamples" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-handling-unseen-levels", - "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-handling-unseen-levels", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding handling unseen levels", - "text": "Target encoding handling unseen levels\n\n\n\n\n\n\nCaution\n\n\nDon’t look at the testing data set. This is done for educational purposes.\n\n\n\n\n\n\nhotel_train |>\n count(agent, .drop = FALSE)\n#> # A tibble: 174 × 2\n#> agent n\n#> <fct> <int>\n#> 1 aaron_marquez 2\n#> 2 aayaat_al_farran 0\n#> 3 alanah_cook 0\n#> 4 alexander_drake 1117\n#> 5 allen_her 1\n#> 6 amirah_christian 0\n#> 7 anas_el_bashir 1\n#> 8 anna_beltran_moreno 0\n#> 9 anna_choi 0\n#> 10 araseli_billy 1\n#> # ℹ 164 more rows\n\n\n\nhotel_test |>\n count(agent, .drop = FALSE)\n#> # A tibble: 174 × 2\n#> agent n\n#> <fct> <int>\n#> 1 aaron_marquez 1\n#> 2 aayaat_al_farran 0\n#> 3 alanah_cook 0\n#> 4 alexander_drake 367\n#> 5 allen_her 1\n#> 6 amirah_christian 0\n#> 7 anas_el_bashir 0\n#> 8 anna_beltran_moreno 0\n#> 9 anna_choi 0\n#> 10 araseli_billy 1\n#> # ℹ 164 more rows" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#fit-our-model-to-the-resamples", + "title": "4 - Evaluating models", + "section": "Fit our model to the resamples", + "text": "Fit our model to the resamples\n\nforested_res <- fit_resamples(forested_wflow, forested_folds)\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 4\n#> splits id .metrics .notes \n#> <list> <chr> <list> <list> \n#> 1 <split [5116/569]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 2 <split [5116/569]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 3 <split [5116/569]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 4 <split [5116/569]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 5 <split [5116/569]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 6 <split [5117/568]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 7 <split [5117/568]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 8 <split [5117/568]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 9 <split [5117/568]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]>\n#> 10 <split [5117/568]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]>" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-handling-unseen-levels-1", - "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-handling-unseen-levels-1", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding handling unseen levels", - "text": "Target encoding handling unseen levels\n\n\nCalculate the global mean of the outcome and use it for cases that aren’t seen in the training data set.\n\n\nmean(hotel_train$avg_price_per_room)\n#> [1] 104.6039\n\n\n\nhotel_train |>\n summarise(\n mean = mean(avg_price_per_room),\n .by = agent\n )\n#> # A tibble: 119 × 2\n#> agent mean\n#> <fct> <dbl>\n#> 1 alexander_drake 144. \n#> 2 kaylae_maxedon 62.5\n#> 3 michael_mcdole 60.9\n#> 4 devin_rivera_borrego 126. \n#> 5 james_richards 78.6\n#> 6 estela_bonilla 41.9\n#> 7 charles_najera 109. \n#> 8 reema_el_tamer 118. \n#> 9 jawhara_al_azad 90.1\n#> 10 not_applicable 84.1\n#> # ℹ 109 more rows" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nforested_res %>%\n collect_metrics()\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.894 10 0.00562 Preprocessor1_Model1\n#> 2 brier_class binary 0.0817 10 0.00434 Preprocessor1_Model1\n#> 3 roc_auc binary 0.951 10 0.00378 Preprocessor1_Model1\n\n\ncollect_metrics() is one of a suite of collect_*() functions that can be used to work with columns of tuning results. Most columns in a tuning result prefixed with . have a corresponding collect_*() function with options for common summaries.\n\n\nWe can reliably measure performance using only the training data 🎉" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#how-do-we-handle-low-counts", - "href": "slides/advanced-05-feature-engineering-part-two.html#how-do-we-handle-low-counts", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "How do we handle low counts?", - "text": "How do we handle low counts?\n\n\nSome of the levels have very low counts. We can’t have the same confidence in those means as the means calculated on high counts.\nWe use the global mean to account for 0 occurrences. Let us adjust the calculated mean by the global mean depending on the counts.\n\n\nhotel_train |>\n summarise(\n mean = mean(avg_price_per_room),\n n = n(),\n .by = agent\n ) |>\n arrange(agent)\n#> # A tibble: 119 × 3\n#> agent mean n\n#> <fct> <dbl> <int>\n#> 1 aaron_marquez 118. 2\n#> 2 alexander_drake 144. 1117\n#> 3 allen_her 65 1\n#> 4 anas_el_bashir 99 1\n#> 5 araseli_billy 40 1\n#> 6 arhab_al_islam 35 7\n#> 7 audray_tucker 76.0 38\n#> 8 bernice_baltierra 71.3 35\n#> 9 betzy_rodriguez 84.0 66\n#> 10 brayan_guerrero 37.5 2\n#> # ℹ 109 more rows" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#comparing-metrics", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#comparing-metrics", + "title": "4 - Evaluating models", + "section": "Comparing metrics ", + "text": "Comparing metrics \nHow do the metrics from resampling compare to the metrics from training and testing?\n\n\n\nforested_res %>%\n collect_metrics() %>% \n select(.metric, mean, n)\n#> # A tibble: 3 × 3\n#> .metric mean n\n#> <chr> <dbl> <int>\n#> 1 accuracy 0.894 10\n#> 2 brier_class 0.0817 10\n#> 3 roc_auc 0.951 10\n\n\nThe ROC AUC previously was\n\n0.97 for the training set\n0.95 for test set\n\n\n\nRemember that:\n⚠️ the training set gives you overly optimistic metrics\n⚠️ the test set is precious" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#partial-pooling", - "href": "slides/advanced-05-feature-engineering-part-two.html#partial-pooling", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Partial pooling", - "text": "Partial pooling\nPartial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage, though." + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-1", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nctrl_forested <- control_resamples(save_pred = TRUE)\nforested_res <- fit_resamples(forested_wflow, forested_folds, control = ctrl_forested)\n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [5116/569]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [5116/569]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [5116/569]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [5116/569]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [5116/569]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [5117/568]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [5117/568]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [5117/568]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [5117/568]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [5117/568]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]> <tibble>" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#partial-pooling-results", - "href": "slides/advanced-05-feature-engineering-part-two.html#partial-pooling-results", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Partial pooling results", - "text": "Partial pooling results" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-2", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-2", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\n# Save the assessment set results\nforested_preds <- collect_predictions(forested_res)\nforested_preds\n#> # A tibble: 5,685 × 7\n#> .pred_class .pred_Yes .pred_No id .row forested .config \n#> <fct> <dbl> <dbl> <chr> <int> <fct> <chr> \n#> 1 Yes 0.5 0.5 Fold01 2 Yes Preprocessor1_Model1\n#> 2 Yes 0.982 0.0178 Fold01 5 Yes Preprocessor1_Model1\n#> 3 No 0.00790 0.992 Fold01 9 No Preprocessor1_Model1\n#> 4 No 0.4 0.6 Fold01 14 No Preprocessor1_Model1\n#> 5 Yes 0.870 0.130 Fold01 18 Yes Preprocessor1_Model1\n#> 6 Yes 0.982 0.0178 Fold01 59 Yes Preprocessor1_Model1\n#> 7 No 0.00790 0.992 Fold01 67 No Preprocessor1_Model1\n#> 8 Yes 0.982 0.0178 Fold01 89 Yes Preprocessor1_Model1\n#> 9 No 0.00790 0.992 Fold01 94 No Preprocessor1_Model1\n#> 10 Yes 0.982 0.0178 Fold01 111 Yes Preprocessor1_Model1\n#> # ℹ 5,675 more rows" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#implentations", - "href": "slides/advanced-05-feature-engineering-part-two.html#implentations", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Implentations", - "text": "Implentations\nWe have described this method solely based on analytical calculations (step_lencode()), but you could arrive at similar numbers using a model-based approach by fitting a no-intercept generalized linear model. A hierarchical version would induce partial pooling.\n\nstep_lencode_glm()\nstep_lencode_bayes()\nstep_lencode_mixed()" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-3", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-3", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nforested_preds %>% \n group_by(id) %>%\n forested_metrics(truth = forested, estimate = .pred_class)\n#> # A tibble: 30 × 4\n#> id .metric .estimator .estimate\n#> <chr> <chr> <chr> <dbl>\n#> 1 Fold01 accuracy binary 0.896\n#> 2 Fold02 accuracy binary 0.859\n#> 3 Fold03 accuracy binary 0.868\n#> 4 Fold04 accuracy binary 0.921\n#> 5 Fold05 accuracy binary 0.900\n#> 6 Fold06 accuracy binary 0.891\n#> 7 Fold07 accuracy binary 0.896\n#> 8 Fold08 accuracy binary 0.903\n#> 9 Fold09 accuracy binary 0.896\n#> 10 Fold10 accuracy binary 0.905\n#> # ℹ 20 more rows" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-in-recipes", - "href": "slides/advanced-05-feature-engineering-part-two.html#target-encoding-in-recipes", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Target encoding in recipes ", - "text": "Target encoding in recipes \n\nrecipe(avg_price_per_room ~ ., data = hotel_train) |>\n step_lencode(\n agent, country, company,\n outcome = vars(\"avg_price_per_room\"), smooth = TRUE,\n ) |>\n prep() |>\n bake(NULL) |>\n select(agent, country, company)\n#> # A tibble: 11,551 × 3\n#> agent country company\n#> <dbl> <dbl> <dbl>\n#> 1 144. 108. 109.\n#> 2 77.2 83.4 109.\n#> 3 61.5 99.9 109.\n#> 4 126. 108. 109.\n#> 5 144. 108. 109.\n#> 6 79.9 73.8 109.\n#> 7 126. 99.9 109.\n#> 8 69.8 108. 109.\n#> 9 126. 99.9 109.\n#> 10 144. 108. 109.\n#> # ℹ 11,541 more rows" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#where-are-the-fitted-models", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#where-are-the-fitted-models", + "title": "4 - Evaluating models", + "section": "Where are the fitted models? ", + "text": "Where are the fitted models? \n\nforested_res\n#> # Resampling results\n#> # 10-fold cross-validation \n#> # A tibble: 10 × 5\n#> splits id .metrics .notes .predictions\n#> <list> <chr> <list> <list> <list> \n#> 1 <split [5116/569]> Fold01 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 2 <split [5116/569]> Fold02 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 3 <split [5116/569]> Fold03 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 4 <split [5116/569]> Fold04 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 5 <split [5116/569]> Fold05 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 6 <split [5117/568]> Fold06 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 7 <split [5117/568]> Fold07 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 8 <split [5117/568]> Fold08 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 9 <split [5117/568]> Fold09 <tibble [3 × 4]> <tibble [0 × 3]> <tibble> \n#> 10 <split [5117/568]> Fold10 <tibble [3 × 4]> <tibble [0 × 3]> <tibble>\n\n\n🗑️" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn-2", - "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn-2", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Your turn", - "text": "Your turn\n\nApply target encoding to the data set, see how it affects different predictors, not just the ones we listed here.\n\nstep_lencode()\nstep_lencode_glm()\nstep_lencode_bayes()\nstep_lencode_mixed()\n\n\n\n\n−+\n03:00" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#bootstrapping", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#bootstrapping", + "title": "4 - Evaluating models", + "section": "Bootstrapping", + "text": "Bootstrapping" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#date-time-variables-1", - "href": "slides/advanced-05-feature-engineering-part-two.html#date-time-variables-1", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Date time variables", - "text": "Date time variables\nHow can we represent the date column arrival_date for our model?\n\nWhen we use a date column in its native format, most models in R convert it to an integer.\n\n\nWe can re-engineer it as:\n\nDays since a reference date\nDay of the week\nMonth\nYear\nIndicators for holidays\n\n\nThe main point is that we try to maximize performance with different versions of the predictors.\nMention that, for the Chicago data, the day or the week features are usually the most important ones in the model.\n\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#bootstrapping-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#bootstrapping-1", + "title": "4 - Evaluating models", + "section": "Bootstrapping ", + "text": "Bootstrapping \n\nset.seed(3214)\nbootstraps(forested_train)\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [5685/2075]> Bootstrap01\n#> 2 <split [5685/2093]> Bootstrap02\n#> 3 <split [5685/2129]> Bootstrap03\n#> 4 <split [5685/2093]> Bootstrap04\n#> 5 <split [5685/2111]> Bootstrap05\n#> 6 <split [5685/2105]> Bootstrap06\n#> 7 <split [5685/2139]> Bootstrap07\n#> 8 <split [5685/2079]> Bootstrap08\n#> 9 <split [5685/2113]> Bootstrap09\n#> 10 <split [5685/2101]> Bootstrap10\n#> # ℹ 15 more rows" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn-3", - "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn-3", - "title": "5 - Feature engineering: splines, target encoding and dates", + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" + }, + { + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-4", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-4", + "title": "4 - Evaluating models", "section": "Your turn", - "text": "Your turn\n\nExplore the arrival_date variable and its relation to avg_price_per_room\nThe lubridate package might provide helpful\n\n\n\n−+\n05:00" + "text": "Your turn\n\nCreate:\n\nMonte Carlo Cross-Validation sets\nvalidation set\n\n(use the reference guide to find the functions)\nDon’t forget to set a seed when you resample!\n\n\n\n−+\n05:00" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#arrival_date-date-features", - "href": "slides/advanced-05-feature-engineering-part-two.html#arrival_date-date-features", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "arrival_date date features", - "text": "arrival_date date features\nusing step_date( features = c(\"year\", \"month\", \"dow\", \"decimal\", \"mday\", \"doy\", \"week\", \"semester\", \"quarter\"))\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\narrival_date\nyear\nmonth\ndow\ndecimal\nmday\ndoy\nweek\nsemester\nquarter\n\n\n\n\n2016-08-30\n2016\nAug\nTue\n2016.661\n30\n243\n35\n2\n3\n\n\n2016-10-22\n2016\nOct\nSat\n2016.806\n22\n296\n43\n2\n4\n\n\n2016-12-17\n2016\nDec\nSat\n2016.959\n17\n352\n51\n2\n4\n\n\n2017-02-13\n2017\nFeb\nMon\n2017.118\n13\n44\n7\n1\n1\n\n\n2017-04-05\n2017\nApr\nWed\n2017.258\n5\n95\n14\n1\n2" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#monte-carlo-cross-validation", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#monte-carlo-cross-validation", + "title": "4 - Evaluating models", + "section": "Monte Carlo Cross-Validation ", + "text": "Monte Carlo Cross-Validation \n\nset.seed(322)\nmc_cv(forested_train, times = 10)\n#> # Monte Carlo cross-validation (0.75/0.25) with 10 resamples \n#> # A tibble: 10 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [4263/1422]> Resample01\n#> 2 <split [4263/1422]> Resample02\n#> 3 <split [4263/1422]> Resample03\n#> 4 <split [4263/1422]> Resample04\n#> 5 <split [4263/1422]> Resample05\n#> 6 <split [4263/1422]> Resample06\n#> 7 <split [4263/1422]> Resample07\n#> 8 <split [4263/1422]> Resample08\n#> 9 <split [4263/1422]> Resample09\n#> 10 <split [4263/1422]> Resample10" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#arrival_date-date-features-1", - "href": "slides/advanced-05-feature-engineering-part-two.html#arrival_date-date-features-1", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "arrival_date date features", - "text": "arrival_date date features\nAdding label = FALSE\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\narrival_date\nyear\nmonth\ndow\ndecimal\nmday\ndoy\nweek\nsemester\nquarter\n\n\n\n\n2016-08-30\n2016\n8\n3\n2016.661\n30\n243\n35\n2\n3\n\n\n2016-10-22\n2016\n10\n7\n2016.806\n22\n296\n43\n2\n4\n\n\n2016-12-17\n2016\n12\n7\n2016.959\n17\n352\n51\n2\n4\n\n\n2017-02-13\n2017\n2\n2\n2017.118\n13\n44\n7\n1\n1\n\n\n2017-04-05\n2017\n4\n4\n2017.258\n5\n95\n14\n1\n2" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#validation-set", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#validation-set", + "title": "4 - Evaluating models", + "section": "Validation set ", + "text": "Validation set \n\nset.seed(853)\nforested_val_split <- initial_validation_split(forested)\nvalidation_set(forested_val_split)\n#> # A tibble: 1 × 2\n#> splits id \n#> <list> <chr> \n#> 1 <split [4264/1421]> validation\n\n\nA validation set is just another type of resample" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#other-recipes-steps", - "href": "slides/advanced-05-feature-engineering-part-two.html#other-recipes-steps", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Other recipes steps", - "text": "Other recipes steps\n\nstep_time()\n\nWorks the same as step_date() but for measurements smaller than day: hour, hour12, am/pm, minute, second, decimal_day.\n\nstep_holiday()\n\nAdds indicators for holidays. See timeDate::listHolidays() for supported holidays." + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#random-forest-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#random-forest-1", + "title": "4 - Evaluating models", + "section": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵", + "text": "Random forest 🌳🌲🌴🌵🌳🌳🌴🌲🌵🌴🌳🌵\n\nEnsemble many decision tree models\nAll the trees vote! 🗳️\nBootstrap aggregating + random predictor sampling\n\n\n\nOften works well without tuning hyperparameters (more on this later!), as long as there are enough trees" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#your-turn-4", - "href": "slides/advanced-05-feature-engineering-part-two.html#your-turn-4", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Your turn", - "text": "Your turn\n\nApply date steps to the arrival_date variable and try to see if we capture anything about avg_price_per_room\n\n\n\n−+\n03:00" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_spec <- rand_forest(trees = 1000, mode = \"classification\")\nrf_spec\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#dates-as-numerics", - "href": "slides/advanced-05-feature-engineering-part-two.html#dates-as-numerics", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "dates as numerics", - "text": "dates as numerics" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#create-a-random-forest-model-1", + "title": "4 - Evaluating models", + "section": "Create a random forest model ", + "text": "Create a random forest model \n\nrf_wflow <- workflow(forested ~ ., rf_spec)\nrf_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Random Forest Model Specification (classification)\n#> \n#> Main Arguments:\n#> trees = 1000\n#> \n#> Computational engine: ranger" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#holidays-as-numerics", - "href": "slides/advanced-05-feature-engineering-part-two.html#holidays-as-numerics", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "holidays as numerics", - "text": "holidays as numerics" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-5", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#your-turn-5", + "title": "4 - Evaluating models", + "section": "Your turn", + "text": "Your turn\n\nUse fit_resamples() and rf_wflow to:\n\nkeep predictions\ncompute metrics\n\n\n\n\n−+\n08:00" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#what-are-the-issues-with-these-features", - "href": "slides/advanced-05-feature-engineering-part-two.html#what-are-the-issues-with-these-features", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "What are the issues with these features?", - "text": "What are the issues with these features?\nThe numeric features make it easy to capture the end or beginning, but harder to do anything more granular.\nThe indicators mostly care about the day itself. No information about the lead-up or aftermath" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-4", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#evaluating-model-performance-4", + "title": "4 - Evaluating models", + "section": "Evaluating model performance ", + "text": "Evaluating model performance \n\nctrl_forested <- control_resamples(save_pred = TRUE)\n\n# Random forest uses random numbers so set the seed first\n\nset.seed(2)\nrf_res <- fit_resamples(rf_wflow, forested_folds, control = ctrl_forested)\ncollect_metrics(rf_res)\n#> # A tibble: 3 × 6\n#> .metric .estimator mean n std_err .config \n#> <chr> <chr> <dbl> <int> <dbl> <chr> \n#> 1 accuracy binary 0.918 10 0.00585 Preprocessor1_Model1\n#> 2 brier_class binary 0.0618 10 0.00337 Preprocessor1_Model1\n#> 3 roc_auc binary 0.972 10 0.00309 Preprocessor1_Model1" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#time-events", - "href": "slides/advanced-05-feature-engineering-part-two.html#time-events", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "time events", - "text": "time events\nUsing extrasteps::step_time_event() and the almanac package, we can create useful time features.\n\nlibrary(almanac)\n\nrule_1 <- weekly() |>\n recur_on_weekdays() |>\n rsetdiff(hol_christmas())\n\nrule_2 <- monthly(since = \"2000-01-01\") |>\n recur_on_interval(3) |>\n recur_on_day_of_month(1)\n\nrule_3 <- yearly(\"1997-06-05\") |>\n recur_on_day_of_week(\"Thursday\") |>\n recur_on_month_of_year(c(\"Jun\", \"July\", \"Aug\"))" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game---status-update-1", + "title": "4 - Evaluating models", + "section": "The whole game - status update", + "text": "The whole game - status update" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_time_event", - "href": "slides/advanced-05-feature-engineering-part-two.html#step_time_event", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "step_time_event() ", - "text": "step_time_event() \nCreate a list of rules (last slide) and pass them to the rules argument of extrasteps::step_time_event()\n\nrules <- list(rule_1 = rule_1, rule_2 = rule_2, rule_3 = rule_3)\n\nrecipe(~arrival_date, data = hotel_rates) |>\n step_time_event(arrival_date, rules = rules)\n\n\nFEAZ" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-final-fit", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-final-fit", + "title": "4 - Evaluating models", + "section": "The final fit ", + "text": "The final fit \nSuppose that we are happy with our random forest model.\nLet’s fit the model on the training set and verify our performance using the test set.\n\nWe’ve shown you fit() and predict() (+ augment()) but there is a shortcut:\n\n# forested_split has train + test info\nfinal_fit <- last_fit(rf_wflow, forested_split) \n\nfinal_fit\n#> # Resampling results\n#> # Manual resampling \n#> # A tibble: 1 × 6\n#> splits id .metrics .notes .predictions .workflow \n#> <list> <chr> <list> <list> <list> <list> \n#> 1 <split [5685/1422]> train/test split <tibble> <tibble> <tibble> <workflow>" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_time_event-as-numerics", - "href": "slides/advanced-05-feature-engineering-part-two.html#step_time_event-as-numerics", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "step_time_event() as numerics ", - "text": "step_time_event() as numerics" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_metrics(final_fit)\n#> # A tibble: 3 × 4\n#> .metric .estimator .estimate .config \n#> <chr> <chr> <dbl> <chr> \n#> 1 accuracy binary 0.911 Preprocessor1_Model1\n#> 2 roc_auc binary 0.970 Preprocessor1_Model1\n#> 3 brier_class binary 0.0652 Preprocessor1_Model1\n\n\nThese are metrics computed with the test set" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#non-indicator-time-events", - "href": "slides/advanced-05-feature-engineering-part-two.html#non-indicator-time-events", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Non-indicator time events", - "text": "Non-indicator time events\nThese features still have the issue that they only attach value to the date itself.\nWe can attach values based on how far we are away from those dates.\n\nstep_date_before()\nstep_date_after()\nstep_date_nearest()" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-1", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-1", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\ncollect_predictions(final_fit)\n#> # A tibble: 1,422 × 7\n#> .pred_class .pred_Yes .pred_No id .row forested .config \n#> <fct> <dbl> <dbl> <chr> <int> <fct> <chr> \n#> 1 Yes 0.822 0.178 train/test split 3 No Preprocessor1…\n#> 2 Yes 0.707 0.293 train/test split 4 Yes Preprocessor1…\n#> 3 No 0.270 0.730 train/test split 7 Yes Preprocessor1…\n#> 4 Yes 0.568 0.432 train/test split 8 Yes Preprocessor1…\n#> 5 Yes 0.554 0.446 train/test split 10 Yes Preprocessor1…\n#> 6 Yes 0.970 0.0297 train/test split 11 Yes Preprocessor1…\n#> 7 Yes 0.963 0.0367 train/test split 12 Yes Preprocessor1…\n#> 8 Yes 0.947 0.0528 train/test split 14 Yes Preprocessor1…\n#> 9 Yes 0.943 0.0573 train/test split 15 Yes Preprocessor1…\n#> 10 Yes 0.977 0.0227 train/test split 19 Yes Preprocessor1…\n#> # ℹ 1,412 more rows" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_date_before", - "href": "slides/advanced-05-feature-engineering-part-two.html#step_date_before", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "step_date_before()", - "text": "step_date_before()" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-2", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#what-is-in-final_fit-2", + "title": "4 - Evaluating models", + "section": "What is in final_fit? ", + "text": "What is in final_fit? \n\nextract_workflow(final_fit)\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Formula\n#> Model: rand_forest()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> forested ~ .\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Ranger result\n#> \n#> Call:\n#> ranger::ranger(x = maybe_data_frame(x), y = y, num.trees = ~1000, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1), probability = TRUE) \n#> \n#> Type: Probability estimation \n#> Number of trees: 1000 \n#> Sample size: 5685 \n#> Number of independent variables: 18 \n#> Mtry: 4 \n#> Target node size: 10 \n#> Variable importance mode: none \n#> Splitrule: gini \n#> OOB prediction error (Brier s.): 0.06153207\n\n\nUse this for prediction on new data, like for deploying" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_date_before---inverse", - "href": "slides/advanced-05-feature-engineering-part-two.html#step_date_before---inverse", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "step_date_before() - inverse", - "text": "step_date_before() - inverse" + "objectID": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game", + "href": "archive/2024-08-posit-conf/intro-04-evaluating-models.html#the-whole-game", + "title": "4 - Evaluating models", + "section": "The whole game", + "text": "The whole game" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_date_after---inverse", - "href": "slides/advanced-05-feature-engineering-part-two.html#step_date_after---inverse", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "step_date_after() - inverse", - "text": "step_date_after() - inverse" + "objectID": "archive/2024-08-posit-conf/intro-06-wrapping-up.html#your-turn", + "href": "archive/2024-08-posit-conf/intro-06-wrapping-up.html#your-turn", + "title": "6 - Wrapping up", + "section": "Your turn", + "text": "Your turn\n\nWhat is one thing you learned that surprised you?\nWhat is one thing you learned that you plan to use?\n\n\n\n−+\n05:00" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#step_date_nearest---inverse", - "href": "slides/advanced-05-feature-engineering-part-two.html#step_date_nearest---inverse", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "step_date_nearest() - inverse", - "text": "step_date_nearest() - inverse" + "objectID": "archive/2024-08-posit-conf/intro-06-wrapping-up.html#resources-to-keep-learning", + "href": "archive/2024-08-posit-conf/intro-06-wrapping-up.html#resources-to-keep-learning", + "title": "6 - Wrapping up", + "section": "Resources to keep learning", + "text": "Resources to keep learning\n\n\nhttps://www.tidymodels.org/\n\n\n\n\nhttps://www.tmwr.org/\n\n\n\n\nhttp://www.feat.engineering/\n\n\n\n\nhttps://smltar.com/\n\n\n\nFollow us on Mastodon and at the tidyverse blog for updates!" }, { - "objectID": "slides/advanced-05-feature-engineering-part-two.html#datetime-features", - "href": "slides/advanced-05-feature-engineering-part-two.html#datetime-features", - "title": "5 - Feature engineering: splines, target encoding and dates", - "section": "Datetime features", - "text": "Datetime features\nAvoid crafting datetime features by hand if at all possible.\nDealing with uneven month lengths, leap days (leap seconds)\nOr tried to define any event that doesn’t land on the same day of the week or date each year.\n\nThe first Sunday after the first full moon on or after the vernal equinox" + "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", + "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#how-can-we-compare-multiple-model-workflows-at-once", + "title": "Extras - workflowsets", + "section": "How can we compare multiple model workflows at once?", + "text": "How can we compare multiple model workflows at once?" }, { - "objectID": "slides/annotations.html#extract-and-update-parameters", - "href": "slides/annotations.html#extract-and-update-parameters", - "title": "Annotations", - "section": "Extract and update parameters", - "text": "Extract and update parameters\nIn about 90% of the cases, the dials function that you use to update the parameter range has the same name as the argument. For example, if you were to update the mtry parameter in a random forests model, the code would look like\n\nparameter_object |> \n update(mtry = mtry(c(1, 100)))\n\nThere are some cases where the parameter function, or its associated values, are different from the argument name.\nFor example, with step_spline_naturall(), we might want to tune the deg_free argument (for the degrees of freedom of a spline function. ). In this case, the argument name is deg_free but we update it with spline_degree().\ndeg_free represents the general concept of degrees of freedom and could be associated with many different things. For example, if we ever had an argument that was the number of degrees of freedom for a \\(t\\) distribution, we would call that argument deg_free.\nFor splines, we probably want a wider range for the degrees of freedom. We made a specialized function called spline_degree() to be used in these cases.\nHow can you tell when this happens? There is a helper function called tunable() and that gives information on how we make the default ranges for parameters. There is a column in these objects names call_info:\n\nlibrary(tidymodels)\nns_tunable <- \n recipe(mpg ~ ., data = mtcars) |> \n step_spline_natural(dis, deg_free = tune()) |> \n tunable()\n\nns_tunable\n#> # A tibble: 1 × 5\n#> name call_info source component component_id \n#> <chr> <list> <chr> <chr> <chr> \n#> 1 deg_free <named list [3]> recipe step_spline_natural spline_natural_P1Tjg\nns_tunable$call_info\n#> [[1]]\n#> [[1]]$pkg\n#> [1] \"dials\"\n#> \n#> [[1]]$fun\n#> [1] \"spline_degree\"\n#> \n#> [[1]]$range\n#> [1] 2 15" + "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set", + "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nwf_set <- workflow_set(list(forested ~ .), list(tree_spec, rf_spec))\nwf_set\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[0]> <list [0]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[0]> <list [0]>" }, { - "objectID": "slides/annotations.html#grid-search", - "href": "slides/annotations.html#grid-search", - "title": "Annotations", - "section": "Grid Search", - "text": "Grid Search\nThere might be some warnings that\n\n“Early stopping occurred at epoch 3 due to numerical overflow of the loss function.”\n\ntorch can be very aggressive about moving in the direction of the gradient. In some cases, it moves to a space where the gradient is not well-behaved (e.g., flat in multiple directions, saddle point, etc).\nThis can cause abnormally large parameter values with magnitudes larger than double precision variables can hold.\nIn this case, brulee stops the optimization and returns the parameters from the last best iteration." + "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", + "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-1", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nwf_set_fit <- wf_set %>%\n workflow_map(\"fit_resamples\", resamples = forested_folds)\nwf_set_fit\n#> # A workflow set/tibble: 2 × 4\n#> wflow_id info option result \n#> <chr> <list> <list> <list> \n#> 1 formula_decision_tree <tibble [1 × 4]> <opts[1]> <rsmp[+]>\n#> 2 formula_rand_forest <tibble [1 × 4]> <opts[1]> <rsmp[+]>" }, { - "objectID": "slides/annotations.html#running-in-parallel", - "href": "slides/annotations.html#running-in-parallel", - "title": "Annotations", - "section": "Running in parallel", - "text": "Running in parallel\nWe usually leave one or two cores available to work with when we run in parallel.\nIf you are using an HPC system, be careful to use only the cores allocated to you.\nAlso, memory is duplicated for each worker. If your main R session is using 1GB of memory, using 10 workers requires 11GB.\nComparing these methods:\n\nThey usually perform about the same.\nfuture is more mature and is geared towards users and developers.\nmirai contains its own queuing system and can be more efficient in allocating work." + "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", + "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#evaluate-a-workflow-set-2", + "title": "Extras - workflowsets", + "section": "Evaluate a workflow set", + "text": "Evaluate a workflow set\n\nwf_set_fit %>%\n rank_results()\n#> # A tibble: 6 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> <chr> <chr> <chr> <dbl> <dbl> <int> <chr> <chr> <int>\n#> 1 formula_rand_fo… Prepro… accura… 0.919 0.00525 10 formula rand… 1\n#> 2 formula_rand_fo… Prepro… brier_… 0.0617 0.00338 10 formula rand… 1\n#> 3 formula_rand_fo… Prepro… roc_auc 0.972 0.00310 10 formula rand… 1\n#> 4 formula_decisio… Prepro… accura… 0.894 0.00562 10 formula deci… 2\n#> 5 formula_decisio… Prepro… brier_… 0.0817 0.00434 10 formula deci… 2\n#> 6 formula_decisio… Prepro… roc_auc 0.951 0.00378 10 formula deci… 2\n\nThe first metric of the metric set is used for ranking. Use rank_metric to change that.\n\nLots more available with workflow sets, like collect_metrics(), autoplot() methods, and more!" }, { - "objectID": "slides/annotations.html#isomap-with-recipes", - "href": "slides/annotations.html#isomap-with-recipes", - "title": "Annotations", - "section": "isomap with recipes", - "text": "isomap with recipes\nThe results from isomap unlike UMAP is not able to be distorted in the same way. Here the number of nieghbors change between the charts. Each of the charts are an application of isomap on the same completely random data. The points are in different locations but none of the charts appear to show any patterns.\n\n#> Warning: The `x` argument of `as_tibble.matrix()` must have unique column\n#> names if `.name_repair` is omitted as of tibble 2.0.0.\n#> ℹ Using compatibility `.name_repair`.\n#> 2025-09-11 12:44:46.88998: Isomap START\n#> 2025-09-11 12:44:46.890359: constructing knn graph\n#> 2025-09-11 12:44:46.901975: calculating geodesic distances\n#> 2025-09-11 12:44:46.903437: Classical Scaling\n#> 2025-09-11 12:44:46.997862: Isomap START\n#> 2025-09-11 12:44:47.000672: constructing knn graph\n#> 2025-09-11 12:44:47.008607: calculating geodesic distances\n#> 2025-09-11 12:44:47.009998: Classical Scaling\n#> 2025-09-11 12:44:47.094436: Isomap START\n#> 2025-09-11 12:44:47.094649: constructing knn graph\n#> 2025-09-11 12:44:47.102851: calculating geodesic distances\n#> 2025-09-11 12:44:47.104791: Classical Scaling\n#> 2025-09-11 12:44:47.191248: Isomap START\n#> 2025-09-11 12:44:47.191422: constructing knn graph\n#> 2025-09-11 12:44:47.199814: calculating geodesic distances\n#> 2025-09-11 12:44:47.202704: Classical Scaling" + "objectID": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#your-turn", + "href": "archive/2024-08-posit-conf/intro-extra-workflowsets.html#your-turn", + "title": "Extras - workflowsets", + "section": "Your turn", + "text": "Your turn\n\nWhen do you think a workflow set would be useful?\nDiscuss with your neighbors!\n\n\n\n−+\n03:00" } ] \ No newline at end of file diff --git a/docs/site_libs/bootstrap/bootstrap-03425fa25cd18b55b1c02e73003bdb3e.min.css b/docs/site_libs/bootstrap/bootstrap-03425fa25cd18b55b1c02e73003bdb3e.min.css deleted file mode 100644 index 9126e153..00000000 --- a/docs/site_libs/bootstrap/bootstrap-03425fa25cd18b55b1c02e73003bdb3e.min.css +++ /dev/null @@ -1,12 +0,0 @@ -/*! - * Bootstrap v5.3.1 (https://getbootstrap.com/) - * Copyright 2011-2023 The Bootstrap Authors - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE) - */@import"https://fonts.googleapis.com/css2?family=Source+Sans+Pro:wght@300;400;700&display=swap";:root,[data-bs-theme=light]{--bs-blue: #2780e3;--bs-indigo: #6610f2;--bs-purple: #613d7c;--bs-pink: #e83e8c;--bs-red: #ff0039;--bs-orange: #f0ad4e;--bs-yellow: #ff7518;--bs-green: #3fb618;--bs-teal: #20c997;--bs-cyan: #9954bb;--bs-black: #000;--bs-white: #fff;--bs-gray: #6c757d;--bs-gray-dark: #343a40;--bs-gray-100: #f8f9fa;--bs-gray-200: #e9ecef;--bs-gray-300: #dee2e6;--bs-gray-400: #ced4da;--bs-gray-500: #adb5bd;--bs-gray-600: #6c757d;--bs-gray-700: #495057;--bs-gray-800: #343a40;--bs-gray-900: #212529;--bs-default: #343a40;--bs-primary: #2780e3;--bs-secondary: #343a40;--bs-success: #3fb618;--bs-info: #9954bb;--bs-warning: #ff7518;--bs-danger: #ff0039;--bs-light: #f8f9fa;--bs-dark: #343a40;--bs-default-rgb: 52, 58, 64;--bs-primary-rgb: 39, 128, 227;--bs-secondary-rgb: 52, 58, 64;--bs-success-rgb: 63, 182, 24;--bs-info-rgb: 153, 84, 187;--bs-warning-rgb: 255, 117, 24;--bs-danger-rgb: 255, 0, 57;--bs-light-rgb: 248, 249, 250;--bs-dark-rgb: 52, 58, 64;--bs-primary-text-emphasis: #10335b;--bs-secondary-text-emphasis: #15171a;--bs-success-text-emphasis: #19490a;--bs-info-text-emphasis: #3d224b;--bs-warning-text-emphasis: #662f0a;--bs-danger-text-emphasis: #660017;--bs-light-text-emphasis: #495057;--bs-dark-text-emphasis: #495057;--bs-primary-bg-subtle: #d4e6f9;--bs-secondary-bg-subtle: #d6d8d9;--bs-success-bg-subtle: #d9f0d1;--bs-info-bg-subtle: #ebddf1;--bs-warning-bg-subtle: #ffe3d1;--bs-danger-bg-subtle: #ffccd7;--bs-light-bg-subtle: #fcfcfd;--bs-dark-bg-subtle: #ced4da;--bs-primary-border-subtle: #a9ccf4;--bs-secondary-border-subtle: #aeb0b3;--bs-success-border-subtle: #b2e2a3;--bs-info-border-subtle: #d6bbe4;--bs-warning-border-subtle: #ffc8a3;--bs-danger-border-subtle: #ff99b0;--bs-light-border-subtle: #e9ecef;--bs-dark-border-subtle: #adb5bd;--bs-white-rgb: 255, 255, 255;--bs-black-rgb: 0, 0, 0;--bs-font-sans-serif: "Source Sans Pro", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";--bs-font-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;--bs-gradient: linear-gradient(180deg, rgba(255, 255, 255, 0.15), rgba(255, 255, 255, 0));--bs-root-font-size: 17px;--bs-body-font-family: "Source Sans Pro", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";--bs-body-font-size:1rem;--bs-body-font-weight: 400;--bs-body-line-height: 1.5;--bs-body-color: #343a40;--bs-body-color-rgb: 52, 58, 64;--bs-body-bg: #fff;--bs-body-bg-rgb: 255, 255, 255;--bs-emphasis-color: #000;--bs-emphasis-color-rgb: 0, 0, 0;--bs-secondary-color: rgba(52, 58, 64, 0.75);--bs-secondary-color-rgb: 52, 58, 64;--bs-secondary-bg: #e9ecef;--bs-secondary-bg-rgb: 233, 236, 239;--bs-tertiary-color: rgba(52, 58, 64, 0.5);--bs-tertiary-color-rgb: 52, 58, 64;--bs-tertiary-bg: #f8f9fa;--bs-tertiary-bg-rgb: 248, 249, 250;--bs-heading-color: inherit;--bs-link-color: #2761e3;--bs-link-color-rgb: 39, 97, 227;--bs-link-decoration: underline;--bs-link-hover-color: #1f4eb6;--bs-link-hover-color-rgb: 31, 78, 182;--bs-code-color: #7d12ba;--bs-highlight-bg: #ffe3d1;--bs-border-width: 1px;--bs-border-style: solid;--bs-border-color: #dee2e6;--bs-border-color-translucent: rgba(0, 0, 0, 0.175);--bs-border-radius: 0.25rem;--bs-border-radius-sm: 0.2em;--bs-border-radius-lg: 0.5rem;--bs-border-radius-xl: 1rem;--bs-border-radius-xxl: 2rem;--bs-border-radius-2xl: var(--bs-border-radius-xxl);--bs-border-radius-pill: 50rem;--bs-box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15);--bs-box-shadow-sm: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.075);--bs-box-shadow-lg: 0 1rem 3rem rgba(0, 0, 0, 0.175);--bs-box-shadow-inset: inset 0 1px 2px rgba(0, 0, 0, 0.075);--bs-focus-ring-width: 0.25rem;--bs-focus-ring-opacity: 0.25;--bs-focus-ring-color: rgba(39, 128, 227, 0.25);--bs-form-valid-color: #3fb618;--bs-form-valid-border-color: #3fb618;--bs-form-invalid-color: #ff0039;--bs-form-invalid-border-color: #ff0039}[data-bs-theme=dark]{color-scheme:dark;--bs-body-color: #dee2e6;--bs-body-color-rgb: 222, 226, 230;--bs-body-bg: #212529;--bs-body-bg-rgb: 33, 37, 41;--bs-emphasis-color: #fff;--bs-emphasis-color-rgb: 255, 255, 255;--bs-secondary-color: rgba(222, 226, 230, 0.75);--bs-secondary-color-rgb: 222, 226, 230;--bs-secondary-bg: #343a40;--bs-secondary-bg-rgb: 52, 58, 64;--bs-tertiary-color: rgba(222, 226, 230, 0.5);--bs-tertiary-color-rgb: 222, 226, 230;--bs-tertiary-bg: #2b3035;--bs-tertiary-bg-rgb: 43, 48, 53;--bs-primary-text-emphasis: #7db3ee;--bs-secondary-text-emphasis: #85898c;--bs-success-text-emphasis: #8cd374;--bs-info-text-emphasis: #c298d6;--bs-warning-text-emphasis: #ffac74;--bs-danger-text-emphasis: #ff6688;--bs-light-text-emphasis: #f8f9fa;--bs-dark-text-emphasis: #dee2e6;--bs-primary-bg-subtle: #081a2d;--bs-secondary-bg-subtle: #0a0c0d;--bs-success-bg-subtle: #0d2405;--bs-info-bg-subtle: #1f1125;--bs-warning-bg-subtle: #331705;--bs-danger-bg-subtle: #33000b;--bs-light-bg-subtle: #343a40;--bs-dark-bg-subtle: #1a1d20;--bs-primary-border-subtle: #174d88;--bs-secondary-border-subtle: #1f2326;--bs-success-border-subtle: #266d0e;--bs-info-border-subtle: #5c3270;--bs-warning-border-subtle: #99460e;--bs-danger-border-subtle: #990022;--bs-light-border-subtle: #495057;--bs-dark-border-subtle: #343a40;--bs-heading-color: inherit;--bs-link-color: #7db3ee;--bs-link-hover-color: #97c2f1;--bs-link-color-rgb: 125, 179, 238;--bs-link-hover-color-rgb: 151, 194, 241;--bs-code-color: white;--bs-border-color: #495057;--bs-border-color-translucent: rgba(255, 255, 255, 0.15);--bs-form-valid-color: #8cd374;--bs-form-valid-border-color: #8cd374;--bs-form-invalid-color: #ff6688;--bs-form-invalid-border-color: #ff6688}*,*::before,*::after{box-sizing:border-box}:root{font-size:var(--bs-root-font-size)}body{margin:0;font-family:var(--bs-body-font-family);font-size:var(--bs-body-font-size);font-weight:var(--bs-body-font-weight);line-height:var(--bs-body-line-height);color:var(--bs-body-color);text-align:var(--bs-body-text-align);background-color:var(--bs-body-bg);-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:rgba(0,0,0,0)}hr{margin:1rem 0;color:inherit;border:0;border-top:1px solid;opacity:.25}h6,.h6,h5,.h5,h4,.h4,h3,.h3,h2,.h2,h1,.h1{margin-top:0;margin-bottom:.5rem;font-weight:400;line-height:1.2;color:var(--bs-heading-color)}h1,.h1{font-size:calc(1.325rem + 0.9vw)}@media(min-width: 1200px){h1,.h1{font-size:2rem}}h2,.h2{font-size:calc(1.29rem + 0.48vw)}@media(min-width: 1200px){h2,.h2{font-size:1.65rem}}h3,.h3{font-size:calc(1.27rem + 0.24vw)}@media(min-width: 1200px){h3,.h3{font-size:1.45rem}}h4,.h4{font-size:1.25rem}h5,.h5{font-size:1.1rem}h6,.h6{font-size:1rem}p{margin-top:0;margin-bottom:1rem}abbr[title]{text-decoration:underline dotted;-webkit-text-decoration:underline dotted;-moz-text-decoration:underline dotted;-ms-text-decoration:underline dotted;-o-text-decoration:underline dotted;cursor:help;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}ol,ul{padding-left:2rem}ol,ul,dl{margin-top:0;margin-bottom:1rem}ol ol,ul ul,ol ul,ul ol{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem;padding:.625rem 1.25rem;border-left:.25rem solid #e9ecef}blockquote p:last-child,blockquote ul:last-child,blockquote ol:last-child{margin-bottom:0}b,strong{font-weight:bolder}small,.small{font-size:0.875em}mark,.mark{padding:.1875em;background-color:var(--bs-highlight-bg)}sub,sup{position:relative;font-size:0.75em;line-height:0;vertical-align:baseline}sub{bottom:-0.25em}sup{top:-0.5em}a{color:rgba(var(--bs-link-color-rgb), var(--bs-link-opacity, 1));text-decoration:underline;-webkit-text-decoration:underline;-moz-text-decoration:underline;-ms-text-decoration:underline;-o-text-decoration:underline}a:hover{--bs-link-color-rgb: var(--bs-link-hover-color-rgb)}a:not([href]):not([class]),a:not([href]):not([class]):hover{color:inherit;text-decoration:none}pre,code,kbd,samp{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:1em}pre{display:block;margin-top:0;margin-bottom:1rem;overflow:auto;font-size:0.875em;color:#000;background-color:#f8f9fa;line-height:1.5;padding:.5rem;border:1px solid var(--bs-border-color, #dee2e6)}pre code{background-color:rgba(0,0,0,0);font-size:inherit;color:inherit;word-break:normal}code{font-size:0.875em;color:var(--bs-code-color);background-color:#f8f9fa;padding:.125rem .25rem;word-wrap:break-word}a>code{color:inherit}kbd{padding:.4rem .4rem;font-size:0.875em;color:#fff;background-color:#343a40}kbd kbd{padding:0;font-size:1em}figure{margin:0 0 1rem}img,svg{vertical-align:middle}table{caption-side:bottom;border-collapse:collapse}caption{padding-top:.5rem;padding-bottom:.5rem;color:rgba(52,58,64,.75);text-align:left}th{text-align:inherit;text-align:-webkit-match-parent}thead,tbody,tfoot,tr,td,th{border-color:inherit;border-style:solid;border-width:0}label{display:inline-block}button{border-radius:0}button:focus:not(:focus-visible){outline:0}input,button,select,optgroup,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}select:disabled{opacity:1}[list]:not([type=date]):not([type=datetime-local]):not([type=month]):not([type=week]):not([type=time])::-webkit-calendar-picker-indicator{display:none !important}button,[type=button],[type=reset],[type=submit]{-webkit-appearance:button}button:not(:disabled),[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled){cursor:pointer}::-moz-focus-inner{padding:0;border-style:none}textarea{resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{float:left;width:100%;padding:0;margin-bottom:.5rem;font-size:calc(1.275rem + 0.3vw);line-height:inherit}@media(min-width: 1200px){legend{font-size:1.5rem}}legend+*{clear:left}::-webkit-datetime-edit-fields-wrapper,::-webkit-datetime-edit-text,::-webkit-datetime-edit-minute,::-webkit-datetime-edit-hour-field,::-webkit-datetime-edit-day-field,::-webkit-datetime-edit-month-field,::-webkit-datetime-edit-year-field{padding:0}::-webkit-inner-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-color-swatch-wrapper{padding:0}::file-selector-button{font:inherit;-webkit-appearance:button}output{display:inline-block}iframe{border:0}summary{display:list-item;cursor:pointer}progress{vertical-align:baseline}[hidden]{display:none !important}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:calc(1.625rem + 4.5vw);font-weight:300;line-height:1.2}@media(min-width: 1200px){.display-1{font-size:5rem}}.display-2{font-size:calc(1.575rem + 3.9vw);font-weight:300;line-height:1.2}@media(min-width: 1200px){.display-2{font-size:4.5rem}}.display-3{font-size:calc(1.525rem + 3.3vw);font-weight:300;line-height:1.2}@media(min-width: 1200px){.display-3{font-size:4rem}}.display-4{font-size:calc(1.475rem + 2.7vw);font-weight:300;line-height:1.2}@media(min-width: 1200px){.display-4{font-size:3.5rem}}.display-5{font-size:calc(1.425rem + 2.1vw);font-weight:300;line-height:1.2}@media(min-width: 1200px){.display-5{font-size:3rem}}.display-6{font-size:calc(1.375rem + 1.5vw);font-weight:300;line-height:1.2}@media(min-width: 1200px){.display-6{font-size:2.5rem}}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:0.875em;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote>:last-child{margin-bottom:0}.blockquote-footer{margin-top:-1rem;margin-bottom:1rem;font-size:0.875em;color:#6c757d}.blockquote-footer::before{content:"— "}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:0.875em;color:rgba(52,58,64,.75)}.container,.container-fluid,.container-xxl,.container-xl,.container-lg,.container-md,.container-sm{--bs-gutter-x: 1.5rem;--bs-gutter-y: 0;width:100%;padding-right:calc(var(--bs-gutter-x)*.5);padding-left:calc(var(--bs-gutter-x)*.5);margin-right:auto;margin-left:auto}@media(min-width: 576px){.container-sm,.container{max-width:540px}}@media(min-width: 768px){.container-md,.container-sm,.container{max-width:720px}}@media(min-width: 992px){.container-lg,.container-md,.container-sm,.container{max-width:960px}}@media(min-width: 1200px){.container-xl,.container-lg,.container-md,.container-sm,.container{max-width:1140px}}@media(min-width: 1400px){.container-xxl,.container-xl,.container-lg,.container-md,.container-sm,.container{max-width:1320px}}:root{--bs-breakpoint-xs: 0;--bs-breakpoint-sm: 576px;--bs-breakpoint-md: 768px;--bs-breakpoint-lg: 992px;--bs-breakpoint-xl: 1200px;--bs-breakpoint-xxl: 1400px}.grid{display:grid;grid-template-rows:repeat(var(--bs-rows, 1), 1fr);grid-template-columns:repeat(var(--bs-columns, 12), 1fr);gap:var(--bs-gap, 1.5rem)}.grid .g-col-1{grid-column:auto/span 1}.grid .g-col-2{grid-column:auto/span 2}.grid .g-col-3{grid-column:auto/span 3}.grid .g-col-4{grid-column:auto/span 4}.grid .g-col-5{grid-column:auto/span 5}.grid .g-col-6{grid-column:auto/span 6}.grid .g-col-7{grid-column:auto/span 7}.grid .g-col-8{grid-column:auto/span 8}.grid .g-col-9{grid-column:auto/span 9}.grid .g-col-10{grid-column:auto/span 10}.grid .g-col-11{grid-column:auto/span 11}.grid .g-col-12{grid-column:auto/span 12}.grid .g-start-1{grid-column-start:1}.grid .g-start-2{grid-column-start:2}.grid .g-start-3{grid-column-start:3}.grid .g-start-4{grid-column-start:4}.grid .g-start-5{grid-column-start:5}.grid .g-start-6{grid-column-start:6}.grid .g-start-7{grid-column-start:7}.grid .g-start-8{grid-column-start:8}.grid .g-start-9{grid-column-start:9}.grid .g-start-10{grid-column-start:10}.grid .g-start-11{grid-column-start:11}@media(min-width: 576px){.grid .g-col-sm-1{grid-column:auto/span 1}.grid .g-col-sm-2{grid-column:auto/span 2}.grid .g-col-sm-3{grid-column:auto/span 3}.grid .g-col-sm-4{grid-column:auto/span 4}.grid .g-col-sm-5{grid-column:auto/span 5}.grid .g-col-sm-6{grid-column:auto/span 6}.grid .g-col-sm-7{grid-column:auto/span 7}.grid .g-col-sm-8{grid-column:auto/span 8}.grid .g-col-sm-9{grid-column:auto/span 9}.grid .g-col-sm-10{grid-column:auto/span 10}.grid .g-col-sm-11{grid-column:auto/span 11}.grid .g-col-sm-12{grid-column:auto/span 12}.grid .g-start-sm-1{grid-column-start:1}.grid .g-start-sm-2{grid-column-start:2}.grid .g-start-sm-3{grid-column-start:3}.grid .g-start-sm-4{grid-column-start:4}.grid .g-start-sm-5{grid-column-start:5}.grid .g-start-sm-6{grid-column-start:6}.grid .g-start-sm-7{grid-column-start:7}.grid .g-start-sm-8{grid-column-start:8}.grid .g-start-sm-9{grid-column-start:9}.grid .g-start-sm-10{grid-column-start:10}.grid .g-start-sm-11{grid-column-start:11}}@media(min-width: 768px){.grid .g-col-md-1{grid-column:auto/span 1}.grid .g-col-md-2{grid-column:auto/span 2}.grid .g-col-md-3{grid-column:auto/span 3}.grid .g-col-md-4{grid-column:auto/span 4}.grid .g-col-md-5{grid-column:auto/span 5}.grid .g-col-md-6{grid-column:auto/span 6}.grid .g-col-md-7{grid-column:auto/span 7}.grid .g-col-md-8{grid-column:auto/span 8}.grid .g-col-md-9{grid-column:auto/span 9}.grid .g-col-md-10{grid-column:auto/span 10}.grid .g-col-md-11{grid-column:auto/span 11}.grid .g-col-md-12{grid-column:auto/span 12}.grid .g-start-md-1{grid-column-start:1}.grid .g-start-md-2{grid-column-start:2}.grid .g-start-md-3{grid-column-start:3}.grid .g-start-md-4{grid-column-start:4}.grid .g-start-md-5{grid-column-start:5}.grid .g-start-md-6{grid-column-start:6}.grid .g-start-md-7{grid-column-start:7}.grid .g-start-md-8{grid-column-start:8}.grid .g-start-md-9{grid-column-start:9}.grid .g-start-md-10{grid-column-start:10}.grid .g-start-md-11{grid-column-start:11}}@media(min-width: 992px){.grid .g-col-lg-1{grid-column:auto/span 1}.grid .g-col-lg-2{grid-column:auto/span 2}.grid .g-col-lg-3{grid-column:auto/span 3}.grid .g-col-lg-4{grid-column:auto/span 4}.grid .g-col-lg-5{grid-column:auto/span 5}.grid .g-col-lg-6{grid-column:auto/span 6}.grid .g-col-lg-7{grid-column:auto/span 7}.grid .g-col-lg-8{grid-column:auto/span 8}.grid .g-col-lg-9{grid-column:auto/span 9}.grid .g-col-lg-10{grid-column:auto/span 10}.grid .g-col-lg-11{grid-column:auto/span 11}.grid .g-col-lg-12{grid-column:auto/span 12}.grid .g-start-lg-1{grid-column-start:1}.grid .g-start-lg-2{grid-column-start:2}.grid .g-start-lg-3{grid-column-start:3}.grid .g-start-lg-4{grid-column-start:4}.grid .g-start-lg-5{grid-column-start:5}.grid .g-start-lg-6{grid-column-start:6}.grid .g-start-lg-7{grid-column-start:7}.grid .g-start-lg-8{grid-column-start:8}.grid .g-start-lg-9{grid-column-start:9}.grid .g-start-lg-10{grid-column-start:10}.grid .g-start-lg-11{grid-column-start:11}}@media(min-width: 1200px){.grid .g-col-xl-1{grid-column:auto/span 1}.grid .g-col-xl-2{grid-column:auto/span 2}.grid .g-col-xl-3{grid-column:auto/span 3}.grid .g-col-xl-4{grid-column:auto/span 4}.grid .g-col-xl-5{grid-column:auto/span 5}.grid .g-col-xl-6{grid-column:auto/span 6}.grid .g-col-xl-7{grid-column:auto/span 7}.grid .g-col-xl-8{grid-column:auto/span 8}.grid .g-col-xl-9{grid-column:auto/span 9}.grid .g-col-xl-10{grid-column:auto/span 10}.grid .g-col-xl-11{grid-column:auto/span 11}.grid .g-col-xl-12{grid-column:auto/span 12}.grid .g-start-xl-1{grid-column-start:1}.grid .g-start-xl-2{grid-column-start:2}.grid .g-start-xl-3{grid-column-start:3}.grid .g-start-xl-4{grid-column-start:4}.grid .g-start-xl-5{grid-column-start:5}.grid .g-start-xl-6{grid-column-start:6}.grid .g-start-xl-7{grid-column-start:7}.grid .g-start-xl-8{grid-column-start:8}.grid .g-start-xl-9{grid-column-start:9}.grid .g-start-xl-10{grid-column-start:10}.grid .g-start-xl-11{grid-column-start:11}}@media(min-width: 1400px){.grid .g-col-xxl-1{grid-column:auto/span 1}.grid .g-col-xxl-2{grid-column:auto/span 2}.grid .g-col-xxl-3{grid-column:auto/span 3}.grid .g-col-xxl-4{grid-column:auto/span 4}.grid .g-col-xxl-5{grid-column:auto/span 5}.grid .g-col-xxl-6{grid-column:auto/span 6}.grid .g-col-xxl-7{grid-column:auto/span 7}.grid .g-col-xxl-8{grid-column:auto/span 8}.grid .g-col-xxl-9{grid-column:auto/span 9}.grid .g-col-xxl-10{grid-column:auto/span 10}.grid .g-col-xxl-11{grid-column:auto/span 11}.grid .g-col-xxl-12{grid-column:auto/span 12}.grid .g-start-xxl-1{grid-column-start:1}.grid .g-start-xxl-2{grid-column-start:2}.grid .g-start-xxl-3{grid-column-start:3}.grid .g-start-xxl-4{grid-column-start:4}.grid .g-start-xxl-5{grid-column-start:5}.grid .g-start-xxl-6{grid-column-start:6}.grid .g-start-xxl-7{grid-column-start:7}.grid .g-start-xxl-8{grid-column-start:8}.grid .g-start-xxl-9{grid-column-start:9}.grid .g-start-xxl-10{grid-column-start:10}.grid .g-start-xxl-11{grid-column-start:11}}.table{--bs-table-color-type: initial;--bs-table-bg-type: initial;--bs-table-color-state: initial;--bs-table-bg-state: initial;--bs-table-color: #343a40;--bs-table-bg: #fff;--bs-table-border-color: #dee2e6;--bs-table-accent-bg: transparent;--bs-table-striped-color: #343a40;--bs-table-striped-bg: rgba(0, 0, 0, 0.05);--bs-table-active-color: #343a40;--bs-table-active-bg: rgba(0, 0, 0, 0.1);--bs-table-hover-color: #343a40;--bs-table-hover-bg: rgba(0, 0, 0, 0.075);width:100%;margin-bottom:1rem;vertical-align:top;border-color:var(--bs-table-border-color)}.table>:not(caption)>*>*{padding:.5rem .5rem;color:var(--bs-table-color-state, var(--bs-table-color-type, var(--bs-table-color)));background-color:var(--bs-table-bg);border-bottom-width:1px;box-shadow:inset 0 0 0 9999px var(--bs-table-bg-state, var(--bs-table-bg-type, var(--bs-table-accent-bg)))}.table>tbody{vertical-align:inherit}.table>thead{vertical-align:bottom}.table-group-divider{border-top:calc(1px*2) solid #9a9da0}.caption-top{caption-side:top}.table-sm>:not(caption)>*>*{padding:.25rem .25rem}.table-bordered>:not(caption)>*{border-width:1px 0}.table-bordered>:not(caption)>*>*{border-width:0 1px}.table-borderless>:not(caption)>*>*{border-bottom-width:0}.table-borderless>:not(:first-child){border-top-width:0}.table-striped>tbody>tr:nth-of-type(odd)>*{--bs-table-color-type: var(--bs-table-striped-color);--bs-table-bg-type: var(--bs-table-striped-bg)}.table-striped-columns>:not(caption)>tr>:nth-child(even){--bs-table-color-type: var(--bs-table-striped-color);--bs-table-bg-type: var(--bs-table-striped-bg)}.table-active{--bs-table-color-state: var(--bs-table-active-color);--bs-table-bg-state: var(--bs-table-active-bg)}.table-hover>tbody>tr:hover>*{--bs-table-color-state: var(--bs-table-hover-color);--bs-table-bg-state: var(--bs-table-hover-bg)}.table-primary{--bs-table-color: #000;--bs-table-bg: #d4e6f9;--bs-table-border-color: #bfcfe0;--bs-table-striped-bg: #c9dbed;--bs-table-striped-color: #000;--bs-table-active-bg: #bfcfe0;--bs-table-active-color: #000;--bs-table-hover-bg: #c4d5e6;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-secondary{--bs-table-color: #000;--bs-table-bg: #d6d8d9;--bs-table-border-color: #c1c2c3;--bs-table-striped-bg: #cbcdce;--bs-table-striped-color: #000;--bs-table-active-bg: #c1c2c3;--bs-table-active-color: #000;--bs-table-hover-bg: #c6c8c9;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-success{--bs-table-color: #000;--bs-table-bg: #d9f0d1;--bs-table-border-color: #c3d8bc;--bs-table-striped-bg: #cee4c7;--bs-table-striped-color: #000;--bs-table-active-bg: #c3d8bc;--bs-table-active-color: #000;--bs-table-hover-bg: #c9dec1;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-info{--bs-table-color: #000;--bs-table-bg: #ebddf1;--bs-table-border-color: #d4c7d9;--bs-table-striped-bg: #dfd2e5;--bs-table-striped-color: #000;--bs-table-active-bg: #d4c7d9;--bs-table-active-color: #000;--bs-table-hover-bg: #d9ccdf;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-warning{--bs-table-color: #000;--bs-table-bg: #ffe3d1;--bs-table-border-color: #e6ccbc;--bs-table-striped-bg: #f2d8c7;--bs-table-striped-color: #000;--bs-table-active-bg: #e6ccbc;--bs-table-active-color: #000;--bs-table-hover-bg: #ecd2c1;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-danger{--bs-table-color: #000;--bs-table-bg: #ffccd7;--bs-table-border-color: #e6b8c2;--bs-table-striped-bg: #f2c2cc;--bs-table-striped-color: #000;--bs-table-active-bg: #e6b8c2;--bs-table-active-color: #000;--bs-table-hover-bg: #ecbdc7;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-light{--bs-table-color: #000;--bs-table-bg: #f8f9fa;--bs-table-border-color: #dfe0e1;--bs-table-striped-bg: #ecedee;--bs-table-striped-color: #000;--bs-table-active-bg: #dfe0e1;--bs-table-active-color: #000;--bs-table-hover-bg: #e5e6e7;--bs-table-hover-color: #000;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-dark{--bs-table-color: #fff;--bs-table-bg: #343a40;--bs-table-border-color: #484e53;--bs-table-striped-bg: #3e444a;--bs-table-striped-color: #fff;--bs-table-active-bg: #484e53;--bs-table-active-color: #fff;--bs-table-hover-bg: #43494e;--bs-table-hover-color: #fff;color:var(--bs-table-color);border-color:var(--bs-table-border-color)}.table-responsive{overflow-x:auto;-webkit-overflow-scrolling:touch}@media(max-width: 575.98px){.table-responsive-sm{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media(max-width: 767.98px){.table-responsive-md{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media(max-width: 991.98px){.table-responsive-lg{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media(max-width: 1199.98px){.table-responsive-xl{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media(max-width: 1399.98px){.table-responsive-xxl{overflow-x:auto;-webkit-overflow-scrolling:touch}}.form-label,.shiny-input-container .control-label{margin-bottom:.5rem}.col-form-label{padding-top:calc(0.375rem + 1px);padding-bottom:calc(0.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(0.5rem + 1px);padding-bottom:calc(0.5rem + 1px);font-size:1.25rem}.col-form-label-sm{padding-top:calc(0.25rem + 1px);padding-bottom:calc(0.25rem + 1px);font-size:0.875rem}.form-text{margin-top:.25rem;font-size:0.875em;color:rgba(52,58,64,.75)}.form-control{display:block;width:100%;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#343a40;appearance:none;-webkit-appearance:none;-moz-appearance:none;-ms-appearance:none;-o-appearance:none;background-color:#fff;background-clip:padding-box;border:1px solid #dee2e6;border-radius:0;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media(prefers-reduced-motion: reduce){.form-control{transition:none}}.form-control[type=file]{overflow:hidden}.form-control[type=file]:not(:disabled):not([readonly]){cursor:pointer}.form-control:focus{color:#343a40;background-color:#fff;border-color:#93c0f1;outline:0;box-shadow:0 0 0 .25rem rgba(39,128,227,.25)}.form-control::-webkit-date-and-time-value{min-width:85px;height:1.5em;margin:0}.form-control::-webkit-datetime-edit{display:block;padding:0}.form-control::placeholder{color:rgba(52,58,64,.75);opacity:1}.form-control:disabled{background-color:#e9ecef;opacity:1}.form-control::file-selector-button{padding:.375rem .75rem;margin:-0.375rem -0.75rem;margin-inline-end:.75rem;color:#343a40;background-color:#f8f9fa;pointer-events:none;border-color:inherit;border-style:solid;border-width:0;border-inline-end-width:1px;border-radius:0;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media(prefers-reduced-motion: reduce){.form-control::file-selector-button{transition:none}}.form-control:hover:not(:disabled):not([readonly])::file-selector-button{background-color:#e9ecef}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;line-height:1.5;color:#343a40;background-color:rgba(0,0,0,0);border:solid rgba(0,0,0,0);border-width:1px 0}.form-control-plaintext:focus{outline:0}.form-control-plaintext.form-control-sm,.form-control-plaintext.form-control-lg{padding-right:0;padding-left:0}.form-control-sm{min-height:calc(1.5em + 0.5rem + calc(1px * 2));padding:.25rem .5rem;font-size:0.875rem}.form-control-sm::file-selector-button{padding:.25rem .5rem;margin:-0.25rem -0.5rem;margin-inline-end:.5rem}.form-control-lg{min-height:calc(1.5em + 1rem + calc(1px * 2));padding:.5rem 1rem;font-size:1.25rem}.form-control-lg::file-selector-button{padding:.5rem 1rem;margin:-0.5rem -1rem;margin-inline-end:1rem}textarea.form-control{min-height:calc(1.5em + 0.75rem + calc(1px * 2))}textarea.form-control-sm{min-height:calc(1.5em + 0.5rem + calc(1px * 2))}textarea.form-control-lg{min-height:calc(1.5em + 1rem + calc(1px * 2))}.form-control-color{width:3rem;height:calc(1.5em + 0.75rem + calc(1px * 2));padding:.375rem}.form-control-color:not(:disabled):not([readonly]){cursor:pointer}.form-control-color::-moz-color-swatch{border:0 !important}.form-control-color::-webkit-color-swatch{border:0 !important}.form-control-color.form-control-sm{height:calc(1.5em + 0.5rem + calc(1px * 2))}.form-control-color.form-control-lg{height:calc(1.5em + 1rem + calc(1px * 2))}.form-select{--bs-form-select-bg-img: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='m2 5 6 6 6-6'/%3e%3c/svg%3e");display:block;width:100%;padding:.375rem 2.25rem .375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#343a40;appearance:none;-webkit-appearance:none;-moz-appearance:none;-ms-appearance:none;-o-appearance:none;background-color:#fff;background-image:var(--bs-form-select-bg-img),var(--bs-form-select-bg-icon, none);background-repeat:no-repeat;background-position:right .75rem center;background-size:16px 12px;border:1px solid #dee2e6;border-radius:0;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media(prefers-reduced-motion: reduce){.form-select{transition:none}}.form-select:focus{border-color:#93c0f1;outline:0;box-shadow:0 0 0 .25rem rgba(39,128,227,.25)}.form-select[multiple],.form-select[size]:not([size="1"]){padding-right:.75rem;background-image:none}.form-select:disabled{background-color:#e9ecef}.form-select:-moz-focusring{color:rgba(0,0,0,0);text-shadow:0 0 0 #343a40}.form-select-sm{padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:0.875rem}.form-select-lg{padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem}[data-bs-theme=dark] .form-select{--bs-form-select-bg-img: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23dee2e6' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='m2 5 6 6 6-6'/%3e%3c/svg%3e")}.form-check,.shiny-input-container .checkbox,.shiny-input-container .radio{display:block;min-height:1.5rem;padding-left:0;margin-bottom:.125rem}.form-check .form-check-input,.form-check .shiny-input-container .checkbox input,.form-check .shiny-input-container .radio input,.shiny-input-container .checkbox .form-check-input,.shiny-input-container .checkbox .shiny-input-container .checkbox input,.shiny-input-container .checkbox .shiny-input-container .radio input,.shiny-input-container .radio .form-check-input,.shiny-input-container .radio .shiny-input-container .checkbox input,.shiny-input-container .radio .shiny-input-container .radio input{float:left;margin-left:0}.form-check-reverse{padding-right:0;padding-left:0;text-align:right}.form-check-reverse .form-check-input{float:right;margin-right:0;margin-left:0}.form-check-input,.shiny-input-container .checkbox input,.shiny-input-container .checkbox-inline input,.shiny-input-container .radio input,.shiny-input-container .radio-inline input{--bs-form-check-bg: #fff;width:1em;height:1em;margin-top:.25em;vertical-align:top;appearance:none;-webkit-appearance:none;-moz-appearance:none;-ms-appearance:none;-o-appearance:none;background-color:var(--bs-form-check-bg);background-image:var(--bs-form-check-bg-image);background-repeat:no-repeat;background-position:center;background-size:contain;border:1px solid #dee2e6;print-color-adjust:exact}.form-check-input[type=radio],.shiny-input-container .checkbox input[type=radio],.shiny-input-container .checkbox-inline input[type=radio],.shiny-input-container .radio input[type=radio],.shiny-input-container .radio-inline input[type=radio]{border-radius:50%}.form-check-input:active,.shiny-input-container .checkbox input:active,.shiny-input-container .checkbox-inline input:active,.shiny-input-container .radio input:active,.shiny-input-container .radio-inline input:active{filter:brightness(90%)}.form-check-input:focus,.shiny-input-container .checkbox input:focus,.shiny-input-container .checkbox-inline input:focus,.shiny-input-container .radio input:focus,.shiny-input-container .radio-inline input:focus{border-color:#93c0f1;outline:0;box-shadow:0 0 0 .25rem rgba(39,128,227,.25)}.form-check-input:checked,.shiny-input-container .checkbox input:checked,.shiny-input-container .checkbox-inline input:checked,.shiny-input-container .radio input:checked,.shiny-input-container .radio-inline input:checked{background-color:#2780e3;border-color:#2780e3}.form-check-input:checked[type=checkbox],.shiny-input-container .checkbox input:checked[type=checkbox],.shiny-input-container .checkbox-inline input:checked[type=checkbox],.shiny-input-container .radio input:checked[type=checkbox],.shiny-input-container .radio-inline input:checked[type=checkbox]{--bs-form-check-bg-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='m6 10 3 3 6-6'/%3e%3c/svg%3e")}.form-check-input:checked[type=radio],.shiny-input-container .checkbox input:checked[type=radio],.shiny-input-container .checkbox-inline input:checked[type=radio],.shiny-input-container .radio input:checked[type=radio],.shiny-input-container .radio-inline input:checked[type=radio]{--bs-form-check-bg-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='2' fill='%23fff'/%3e%3c/svg%3e")}.form-check-input[type=checkbox]:indeterminate,.shiny-input-container .checkbox input[type=checkbox]:indeterminate,.shiny-input-container .checkbox-inline input[type=checkbox]:indeterminate,.shiny-input-container .radio input[type=checkbox]:indeterminate,.shiny-input-container .radio-inline input[type=checkbox]:indeterminate{background-color:#2780e3;border-color:#2780e3;--bs-form-check-bg-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='M6 10h8'/%3e%3c/svg%3e")}.form-check-input:disabled,.shiny-input-container .checkbox input:disabled,.shiny-input-container .checkbox-inline input:disabled,.shiny-input-container .radio input:disabled,.shiny-input-container .radio-inline input:disabled{pointer-events:none;filter:none;opacity:.5}.form-check-input[disabled]~.form-check-label,.form-check-input[disabled]~span,.form-check-input:disabled~.form-check-label,.form-check-input:disabled~span,.shiny-input-container .checkbox input[disabled]~.form-check-label,.shiny-input-container .checkbox input[disabled]~span,.shiny-input-container .checkbox input:disabled~.form-check-label,.shiny-input-container .checkbox input:disabled~span,.shiny-input-container .checkbox-inline input[disabled]~.form-check-label,.shiny-input-container .checkbox-inline input[disabled]~span,.shiny-input-container .checkbox-inline input:disabled~.form-check-label,.shiny-input-container .checkbox-inline input:disabled~span,.shiny-input-container .radio input[disabled]~.form-check-label,.shiny-input-container .radio input[disabled]~span,.shiny-input-container .radio input:disabled~.form-check-label,.shiny-input-container .radio input:disabled~span,.shiny-input-container .radio-inline input[disabled]~.form-check-label,.shiny-input-container .radio-inline input[disabled]~span,.shiny-input-container .radio-inline input:disabled~.form-check-label,.shiny-input-container .radio-inline input:disabled~span{cursor:default;opacity:.5}.form-check-label,.shiny-input-container .checkbox label,.shiny-input-container .checkbox-inline label,.shiny-input-container .radio label,.shiny-input-container .radio-inline label{cursor:pointer}.form-switch{padding-left:2.5em}.form-switch .form-check-input{--bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%280, 0, 0, 0.25%29'/%3e%3c/svg%3e");width:2em;margin-left:-2.5em;background-image:var(--bs-form-switch-bg);background-position:left center;transition:background-position .15s ease-in-out}@media(prefers-reduced-motion: reduce){.form-switch .form-check-input{transition:none}}.form-switch .form-check-input:focus{--bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%2393c0f1'/%3e%3c/svg%3e")}.form-switch .form-check-input:checked{background-position:right center;--bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.form-switch.form-check-reverse{padding-right:2.5em;padding-left:0}.form-switch.form-check-reverse .form-check-input{margin-right:-2.5em;margin-left:0}.form-check-inline{display:inline-block;margin-right:1rem}.btn-check{position:absolute;clip:rect(0, 0, 0, 0);pointer-events:none}.btn-check[disabled]+.btn,.btn-check:disabled+.btn{pointer-events:none;filter:none;opacity:.65}[data-bs-theme=dark] .form-switch .form-check-input:not(:checked):not(:focus){--bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%28255, 255, 255, 0.25%29'/%3e%3c/svg%3e")}.form-range{width:100%;height:1.5rem;padding:0;appearance:none;-webkit-appearance:none;-moz-appearance:none;-ms-appearance:none;-o-appearance:none;background-color:rgba(0,0,0,0)}.form-range:focus{outline:0}.form-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem rgba(39,128,227,.25)}.form-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem rgba(39,128,227,.25)}.form-range::-moz-focus-outer{border:0}.form-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-0.25rem;appearance:none;-webkit-appearance:none;-moz-appearance:none;-ms-appearance:none;-o-appearance:none;background-color:#2780e3;border:0;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media(prefers-reduced-motion: reduce){.form-range::-webkit-slider-thumb{transition:none}}.form-range::-webkit-slider-thumb:active{background-color:#bed9f7}.form-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:rgba(0,0,0,0);cursor:pointer;background-color:#f8f9fa;border-color:rgba(0,0,0,0)}.form-range::-moz-range-thumb{width:1rem;height:1rem;appearance:none;-webkit-appearance:none;-moz-appearance:none;-ms-appearance:none;-o-appearance:none;background-color:#2780e3;border:0;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media(prefers-reduced-motion: reduce){.form-range::-moz-range-thumb{transition:none}}.form-range::-moz-range-thumb:active{background-color:#bed9f7}.form-range::-moz-range-track{width:100%;height:.5rem;color:rgba(0,0,0,0);cursor:pointer;background-color:#f8f9fa;border-color:rgba(0,0,0,0)}.form-range:disabled{pointer-events:none}.form-range:disabled::-webkit-slider-thumb{background-color:rgba(52,58,64,.75)}.form-range:disabled::-moz-range-thumb{background-color:rgba(52,58,64,.75)}.form-floating{position:relative}.form-floating>.form-control,.form-floating>.form-control-plaintext,.form-floating>.form-select{height:calc(3.5rem + calc(1px * 2));min-height:calc(3.5rem + calc(1px * 2));line-height:1.25}.form-floating>label{position:absolute;top:0;left:0;z-index:2;height:100%;padding:1rem .75rem;overflow:hidden;text-align:start;text-overflow:ellipsis;white-space:nowrap;pointer-events:none;border:1px solid rgba(0,0,0,0);transform-origin:0 0;transition:opacity .1s ease-in-out,transform .1s ease-in-out}@media(prefers-reduced-motion: reduce){.form-floating>label{transition:none}}.form-floating>.form-control,.form-floating>.form-control-plaintext{padding:1rem .75rem}.form-floating>.form-control::placeholder,.form-floating>.form-control-plaintext::placeholder{color:rgba(0,0,0,0)}.form-floating>.form-control:focus,.form-floating>.form-control:not(:placeholder-shown),.form-floating>.form-control-plaintext:focus,.form-floating>.form-control-plaintext:not(:placeholder-shown){padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:-webkit-autofill,.form-floating>.form-control-plaintext:-webkit-autofill{padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-select{padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:focus~label,.form-floating>.form-control:not(:placeholder-shown)~label,.form-floating>.form-control-plaintext~label,.form-floating>.form-select~label{color:rgba(var(--bs-body-color-rgb), 0.65);transform:scale(0.85) translateY(-0.5rem) translateX(0.15rem)}.form-floating>.form-control:focus~label::after,.form-floating>.form-control:not(:placeholder-shown)~label::after,.form-floating>.form-control-plaintext~label::after,.form-floating>.form-select~label::after{position:absolute;inset:1rem .375rem;z-index:-1;height:1.5em;content:"";background-color:#fff}.form-floating>.form-control:-webkit-autofill~label{color:rgba(var(--bs-body-color-rgb), 0.65);transform:scale(0.85) translateY(-0.5rem) translateX(0.15rem)}.form-floating>.form-control-plaintext~label{border-width:1px 0}.form-floating>:disabled~label,.form-floating>.form-control:disabled~label{color:#6c757d}.form-floating>:disabled~label::after,.form-floating>.form-control:disabled~label::after{background-color:#e9ecef}.input-group{position:relative;display:flex;display:-webkit-flex;flex-wrap:wrap;-webkit-flex-wrap:wrap;align-items:stretch;-webkit-align-items:stretch;width:100%}.input-group>.form-control,.input-group>.form-select,.input-group>.form-floating{position:relative;flex:1 1 auto;-webkit-flex:1 1 auto;width:1%;min-width:0}.input-group>.form-control:focus,.input-group>.form-select:focus,.input-group>.form-floating:focus-within{z-index:5}.input-group .btn{position:relative;z-index:2}.input-group .btn:focus{z-index:5}.input-group-text{display:flex;display:-webkit-flex;align-items:center;-webkit-align-items:center;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#343a40;text-align:center;white-space:nowrap;background-color:#f8f9fa;border:1px solid #dee2e6}.input-group-lg>.form-control,.input-group-lg>.form-select,.input-group-lg>.input-group-text,.input-group-lg>.btn{padding:.5rem 1rem;font-size:1.25rem}.input-group-sm>.form-control,.input-group-sm>.form-select,.input-group-sm>.input-group-text,.input-group-sm>.btn{padding:.25rem .5rem;font-size:0.875rem}.input-group-lg>.form-select,.input-group-sm>.form-select{padding-right:3rem}.input-group>:not(:first-child):not(.dropdown-menu):not(.valid-tooltip):not(.valid-feedback):not(.invalid-tooltip):not(.invalid-feedback){margin-left:calc(1px*-1)}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:0.875em;color:#3fb618}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:0.875rem;color:#fff;background-color:#3fb618}.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip,.is-valid~.valid-feedback,.is-valid~.valid-tooltip{display:block}.was-validated .form-control:valid,.form-control.is-valid{border-color:#3fb618;padding-right:calc(1.5em + 0.75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%233fb618' d='M2.3 6.73.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(0.375em + 0.1875rem) center;background-size:calc(0.75em + 0.375rem) calc(0.75em + 0.375rem)}.was-validated .form-control:valid:focus,.form-control.is-valid:focus{border-color:#3fb618;box-shadow:0 0 0 .25rem rgba(63,182,24,.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + 0.75rem);background-position:top calc(0.375em + 0.1875rem) right calc(0.375em + 0.1875rem)}.was-validated .form-select:valid,.form-select.is-valid{border-color:#3fb618}.was-validated .form-select:valid:not([multiple]):not([size]),.was-validated .form-select:valid:not([multiple])[size="1"],.form-select.is-valid:not([multiple]):not([size]),.form-select.is-valid:not([multiple])[size="1"]{--bs-form-select-bg-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%233fb618' d='M2.3 6.73.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");padding-right:4.125rem;background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(0.75em + 0.375rem) calc(0.75em + 0.375rem)}.was-validated .form-select:valid:focus,.form-select.is-valid:focus{border-color:#3fb618;box-shadow:0 0 0 .25rem rgba(63,182,24,.25)}.was-validated .form-control-color:valid,.form-control-color.is-valid{width:calc(3rem + calc(1.5em + 0.75rem))}.was-validated .form-check-input:valid,.form-check-input.is-valid{border-color:#3fb618}.was-validated .form-check-input:valid:checked,.form-check-input.is-valid:checked{background-color:#3fb618}.was-validated .form-check-input:valid:focus,.form-check-input.is-valid:focus{box-shadow:0 0 0 .25rem rgba(63,182,24,.25)}.was-validated .form-check-input:valid~.form-check-label,.form-check-input.is-valid~.form-check-label{color:#3fb618}.form-check-inline .form-check-input~.valid-feedback{margin-left:.5em}.was-validated .input-group>.form-control:not(:focus):valid,.input-group>.form-control:not(:focus).is-valid,.was-validated .input-group>.form-select:not(:focus):valid,.input-group>.form-select:not(:focus).is-valid,.was-validated .input-group>.form-floating:not(:focus-within):valid,.input-group>.form-floating:not(:focus-within).is-valid{z-index:3}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:0.875em;color:#ff0039}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:0.875rem;color:#fff;background-color:#ff0039}.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip,.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip{display:block}.was-validated .form-control:invalid,.form-control.is-invalid{border-color:#ff0039;padding-right:calc(1.5em + 0.75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23ff0039'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23ff0039' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(0.375em + 0.1875rem) center;background-size:calc(0.75em + 0.375rem) calc(0.75em + 0.375rem)}.was-validated .form-control:invalid:focus,.form-control.is-invalid:focus{border-color:#ff0039;box-shadow:0 0 0 .25rem rgba(255,0,57,.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + 0.75rem);background-position:top calc(0.375em + 0.1875rem) right calc(0.375em + 0.1875rem)}.was-validated .form-select:invalid,.form-select.is-invalid{border-color:#ff0039}.was-validated .form-select:invalid:not([multiple]):not([size]),.was-validated .form-select:invalid:not([multiple])[size="1"],.form-select.is-invalid:not([multiple]):not([size]),.form-select.is-invalid:not([multiple])[size="1"]{--bs-form-select-bg-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23ff0039'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23ff0039' stroke='none'/%3e%3c/svg%3e");padding-right:4.125rem;background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(0.75em + 0.375rem) calc(0.75em + 0.375rem)}.was-validated .form-select:invalid:focus,.form-select.is-invalid:focus{border-color:#ff0039;box-shadow:0 0 0 .25rem rgba(255,0,57,.25)}.was-validated .form-control-color:invalid,.form-control-color.is-invalid{width:calc(3rem + calc(1.5em + 0.75rem))}.was-validated .form-check-input:invalid,.form-check-input.is-invalid{border-color:#ff0039}.was-validated .form-check-input:invalid:checked,.form-check-input.is-invalid:checked{background-color:#ff0039}.was-validated .form-check-input:invalid:focus,.form-check-input.is-invalid:focus{box-shadow:0 0 0 .25rem rgba(255,0,57,.25)}.was-validated .form-check-input:invalid~.form-check-label,.form-check-input.is-invalid~.form-check-label{color:#ff0039}.form-check-inline .form-check-input~.invalid-feedback{margin-left:.5em}.was-validated .input-group>.form-control:not(:focus):invalid,.input-group>.form-control:not(:focus).is-invalid,.was-validated .input-group>.form-select:not(:focus):invalid,.input-group>.form-select:not(:focus).is-invalid,.was-validated .input-group>.form-floating:not(:focus-within):invalid,.input-group>.form-floating:not(:focus-within).is-invalid{z-index:4}.btn{--bs-btn-padding-x: 0.75rem;--bs-btn-padding-y: 0.375rem;--bs-btn-font-family: ;--bs-btn-font-size:1rem;--bs-btn-font-weight: 400;--bs-btn-line-height: 1.5;--bs-btn-color: #343a40;--bs-btn-bg: transparent;--bs-btn-border-width: 1px;--bs-btn-border-color: transparent;--bs-btn-border-radius: 0.25rem;--bs-btn-hover-border-color: transparent;--bs-btn-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.15), 0 1px 1px rgba(0, 0, 0, 0.075);--bs-btn-disabled-opacity: 0.65;--bs-btn-focus-box-shadow: 0 0 0 0.25rem rgba(var(--bs-btn-focus-shadow-rgb), .5);display:inline-block;padding:var(--bs-btn-padding-y) var(--bs-btn-padding-x);font-family:var(--bs-btn-font-family);font-size:var(--bs-btn-font-size);font-weight:var(--bs-btn-font-weight);line-height:var(--bs-btn-line-height);color:var(--bs-btn-color);text-align:center;text-decoration:none;-webkit-text-decoration:none;-moz-text-decoration:none;-ms-text-decoration:none;-o-text-decoration:none;vertical-align:middle;cursor:pointer;user-select:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;-o-user-select:none;border:var(--bs-btn-border-width) solid var(--bs-btn-border-color);background-color:var(--bs-btn-bg);transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media(prefers-reduced-motion: reduce){.btn{transition:none}}.btn:hover{color:var(--bs-btn-hover-color);background-color:var(--bs-btn-hover-bg);border-color:var(--bs-btn-hover-border-color)}.btn-check+.btn:hover{color:var(--bs-btn-color);background-color:var(--bs-btn-bg);border-color:var(--bs-btn-border-color)}.btn:focus-visible{color:var(--bs-btn-hover-color);background-color:var(--bs-btn-hover-bg);border-color:var(--bs-btn-hover-border-color);outline:0;box-shadow:var(--bs-btn-focus-box-shadow)}.btn-check:focus-visible+.btn{border-color:var(--bs-btn-hover-border-color);outline:0;box-shadow:var(--bs-btn-focus-box-shadow)}.btn-check:checked+.btn,:not(.btn-check)+.btn:active,.btn:first-child:active,.btn.active,.btn.show{color:var(--bs-btn-active-color);background-color:var(--bs-btn-active-bg);border-color:var(--bs-btn-active-border-color)}.btn-check:checked+.btn:focus-visible,:not(.btn-check)+.btn:active:focus-visible,.btn:first-child:active:focus-visible,.btn.active:focus-visible,.btn.show:focus-visible{box-shadow:var(--bs-btn-focus-box-shadow)}.btn:disabled,.btn.disabled,fieldset:disabled .btn{color:var(--bs-btn-disabled-color);pointer-events:none;background-color:var(--bs-btn-disabled-bg);border-color:var(--bs-btn-disabled-border-color);opacity:var(--bs-btn-disabled-opacity)}.btn-default{--bs-btn-color: #fff;--bs-btn-bg: #343a40;--bs-btn-border-color: #343a40;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #2c3136;--bs-btn-hover-border-color: #2a2e33;--bs-btn-focus-shadow-rgb: 82, 88, 93;--bs-btn-active-color: #fff;--bs-btn-active-bg: #2a2e33;--bs-btn-active-border-color: #272c30;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #343a40;--bs-btn-disabled-border-color: #343a40}.btn-primary{--bs-btn-color: #fff;--bs-btn-bg: #2780e3;--bs-btn-border-color: #2780e3;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #216dc1;--bs-btn-hover-border-color: #1f66b6;--bs-btn-focus-shadow-rgb: 71, 147, 231;--bs-btn-active-color: #fff;--bs-btn-active-bg: #1f66b6;--bs-btn-active-border-color: #1d60aa;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #2780e3;--bs-btn-disabled-border-color: #2780e3}.btn-secondary{--bs-btn-color: #fff;--bs-btn-bg: #343a40;--bs-btn-border-color: #343a40;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #2c3136;--bs-btn-hover-border-color: #2a2e33;--bs-btn-focus-shadow-rgb: 82, 88, 93;--bs-btn-active-color: #fff;--bs-btn-active-bg: #2a2e33;--bs-btn-active-border-color: #272c30;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #343a40;--bs-btn-disabled-border-color: #343a40}.btn-success{--bs-btn-color: #fff;--bs-btn-bg: #3fb618;--bs-btn-border-color: #3fb618;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #369b14;--bs-btn-hover-border-color: #329213;--bs-btn-focus-shadow-rgb: 92, 193, 59;--bs-btn-active-color: #fff;--bs-btn-active-bg: #329213;--bs-btn-active-border-color: #2f8912;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #3fb618;--bs-btn-disabled-border-color: #3fb618}.btn-info{--bs-btn-color: #fff;--bs-btn-bg: #9954bb;--bs-btn-border-color: #9954bb;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #82479f;--bs-btn-hover-border-color: #7a4396;--bs-btn-focus-shadow-rgb: 168, 110, 197;--bs-btn-active-color: #fff;--bs-btn-active-bg: #7a4396;--bs-btn-active-border-color: #733f8c;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #9954bb;--bs-btn-disabled-border-color: #9954bb}.btn-warning{--bs-btn-color: #fff;--bs-btn-bg: #ff7518;--bs-btn-border-color: #ff7518;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #d96314;--bs-btn-hover-border-color: #cc5e13;--bs-btn-focus-shadow-rgb: 255, 138, 59;--bs-btn-active-color: #fff;--bs-btn-active-bg: #cc5e13;--bs-btn-active-border-color: #bf5812;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #ff7518;--bs-btn-disabled-border-color: #ff7518}.btn-danger{--bs-btn-color: #fff;--bs-btn-bg: #ff0039;--bs-btn-border-color: #ff0039;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #d90030;--bs-btn-hover-border-color: #cc002e;--bs-btn-focus-shadow-rgb: 255, 38, 87;--bs-btn-active-color: #fff;--bs-btn-active-bg: #cc002e;--bs-btn-active-border-color: #bf002b;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #ff0039;--bs-btn-disabled-border-color: #ff0039}.btn-light{--bs-btn-color: #000;--bs-btn-bg: #f8f9fa;--bs-btn-border-color: #f8f9fa;--bs-btn-hover-color: #000;--bs-btn-hover-bg: #d3d4d5;--bs-btn-hover-border-color: #c6c7c8;--bs-btn-focus-shadow-rgb: 211, 212, 213;--bs-btn-active-color: #000;--bs-btn-active-bg: #c6c7c8;--bs-btn-active-border-color: #babbbc;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #000;--bs-btn-disabled-bg: #f8f9fa;--bs-btn-disabled-border-color: #f8f9fa}.btn-dark{--bs-btn-color: #fff;--bs-btn-bg: #343a40;--bs-btn-border-color: #343a40;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #52585d;--bs-btn-hover-border-color: #484e53;--bs-btn-focus-shadow-rgb: 82, 88, 93;--bs-btn-active-color: #fff;--bs-btn-active-bg: #5d6166;--bs-btn-active-border-color: #484e53;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #343a40;--bs-btn-disabled-border-color: #343a40}.btn-outline-default{--bs-btn-color: #343a40;--bs-btn-border-color: #343a40;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #343a40;--bs-btn-hover-border-color: #343a40;--bs-btn-focus-shadow-rgb: 52, 58, 64;--bs-btn-active-color: #fff;--bs-btn-active-bg: #343a40;--bs-btn-active-border-color: #343a40;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #343a40;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #343a40;--bs-btn-bg: transparent;--bs-gradient: none}.btn-outline-primary{--bs-btn-color: #2780e3;--bs-btn-border-color: #2780e3;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #2780e3;--bs-btn-hover-border-color: #2780e3;--bs-btn-focus-shadow-rgb: 39, 128, 227;--bs-btn-active-color: #fff;--bs-btn-active-bg: #2780e3;--bs-btn-active-border-color: #2780e3;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #2780e3;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #2780e3;--bs-btn-bg: transparent;--bs-gradient: none}.btn-outline-secondary{--bs-btn-color: #343a40;--bs-btn-border-color: #343a40;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #343a40;--bs-btn-hover-border-color: #343a40;--bs-btn-focus-shadow-rgb: 52, 58, 64;--bs-btn-active-color: #fff;--bs-btn-active-bg: #343a40;--bs-btn-active-border-color: #343a40;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #343a40;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #343a40;--bs-btn-bg: transparent;--bs-gradient: none}.btn-outline-success{--bs-btn-color: #3fb618;--bs-btn-border-color: #3fb618;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #3fb618;--bs-btn-hover-border-color: #3fb618;--bs-btn-focus-shadow-rgb: 63, 182, 24;--bs-btn-active-color: #fff;--bs-btn-active-bg: #3fb618;--bs-btn-active-border-color: #3fb618;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #3fb618;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #3fb618;--bs-btn-bg: transparent;--bs-gradient: none}.btn-outline-info{--bs-btn-color: #9954bb;--bs-btn-border-color: #9954bb;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #9954bb;--bs-btn-hover-border-color: #9954bb;--bs-btn-focus-shadow-rgb: 153, 84, 187;--bs-btn-active-color: #fff;--bs-btn-active-bg: #9954bb;--bs-btn-active-border-color: #9954bb;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #9954bb;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #9954bb;--bs-btn-bg: transparent;--bs-gradient: none}.btn-outline-warning{--bs-btn-color: #ff7518;--bs-btn-border-color: #ff7518;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #ff7518;--bs-btn-hover-border-color: #ff7518;--bs-btn-focus-shadow-rgb: 255, 117, 24;--bs-btn-active-color: #fff;--bs-btn-active-bg: #ff7518;--bs-btn-active-border-color: #ff7518;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #ff7518;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #ff7518;--bs-btn-bg: transparent;--bs-gradient: none}.btn-outline-danger{--bs-btn-color: #ff0039;--bs-btn-border-color: #ff0039;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #ff0039;--bs-btn-hover-border-color: #ff0039;--bs-btn-focus-shadow-rgb: 255, 0, 57;--bs-btn-active-color: #fff;--bs-btn-active-bg: #ff0039;--bs-btn-active-border-color: #ff0039;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #ff0039;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #ff0039;--bs-btn-bg: transparent;--bs-gradient: none}.btn-outline-light{--bs-btn-color: #f8f9fa;--bs-btn-border-color: #f8f9fa;--bs-btn-hover-color: #000;--bs-btn-hover-bg: #f8f9fa;--bs-btn-hover-border-color: #f8f9fa;--bs-btn-focus-shadow-rgb: 248, 249, 250;--bs-btn-active-color: #000;--bs-btn-active-bg: #f8f9fa;--bs-btn-active-border-color: #f8f9fa;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #f8f9fa;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #f8f9fa;--bs-btn-bg: transparent;--bs-gradient: none}.btn-outline-dark{--bs-btn-color: #343a40;--bs-btn-border-color: #343a40;--bs-btn-hover-color: #fff;--bs-btn-hover-bg: #343a40;--bs-btn-hover-border-color: #343a40;--bs-btn-focus-shadow-rgb: 52, 58, 64;--bs-btn-active-color: #fff;--bs-btn-active-bg: #343a40;--bs-btn-active-border-color: #343a40;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #343a40;--bs-btn-disabled-bg: transparent;--bs-btn-disabled-border-color: #343a40;--bs-btn-bg: transparent;--bs-gradient: none}.btn-link{--bs-btn-font-weight: 400;--bs-btn-color: #2761e3;--bs-btn-bg: transparent;--bs-btn-border-color: transparent;--bs-btn-hover-color: #1f4eb6;--bs-btn-hover-border-color: transparent;--bs-btn-active-color: #1f4eb6;--bs-btn-active-border-color: transparent;--bs-btn-disabled-color: #6c757d;--bs-btn-disabled-border-color: transparent;--bs-btn-box-shadow: 0 0 0 #000;--bs-btn-focus-shadow-rgb: 71, 121, 231;text-decoration:underline;-webkit-text-decoration:underline;-moz-text-decoration:underline;-ms-text-decoration:underline;-o-text-decoration:underline}.btn-link:focus-visible{color:var(--bs-btn-color)}.btn-link:hover{color:var(--bs-btn-hover-color)}.btn-lg,.btn-group-lg>.btn{--bs-btn-padding-y: 0.5rem;--bs-btn-padding-x: 1rem;--bs-btn-font-size:1.25rem;--bs-btn-border-radius: 0.5rem}.btn-sm,.btn-group-sm>.btn{--bs-btn-padding-y: 0.25rem;--bs-btn-padding-x: 0.5rem;--bs-btn-font-size:0.875rem;--bs-btn-border-radius: 0.2em}.fade{transition:opacity .15s linear}@media(prefers-reduced-motion: reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{height:0;overflow:hidden;transition:height .2s ease}@media(prefers-reduced-motion: reduce){.collapsing{transition:none}}.collapsing.collapse-horizontal{width:0;height:auto;transition:width .35s ease}@media(prefers-reduced-motion: reduce){.collapsing.collapse-horizontal{transition:none}}.dropup,.dropend,.dropdown,.dropstart,.dropup-center,.dropdown-center{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid rgba(0,0,0,0);border-bottom:0;border-left:.3em solid rgba(0,0,0,0)}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{--bs-dropdown-zindex: 1000;--bs-dropdown-min-width: 10rem;--bs-dropdown-padding-x: 0;--bs-dropdown-padding-y: 0.5rem;--bs-dropdown-spacer: 0.125rem;--bs-dropdown-font-size:1rem;--bs-dropdown-color: #343a40;--bs-dropdown-bg: #fff;--bs-dropdown-border-color: rgba(0, 0, 0, 0.175);--bs-dropdown-border-radius: 0.25rem;--bs-dropdown-border-width: 1px;--bs-dropdown-inner-border-radius: calc(0.25rem - 1px);--bs-dropdown-divider-bg: rgba(0, 0, 0, 0.175);--bs-dropdown-divider-margin-y: 0.5rem;--bs-dropdown-box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15);--bs-dropdown-link-color: #343a40;--bs-dropdown-link-hover-color: #343a40;--bs-dropdown-link-hover-bg: #f8f9fa;--bs-dropdown-link-active-color: #fff;--bs-dropdown-link-active-bg: #2780e3;--bs-dropdown-link-disabled-color: rgba(52, 58, 64, 0.5);--bs-dropdown-item-padding-x: 1rem;--bs-dropdown-item-padding-y: 0.25rem;--bs-dropdown-header-color: #6c757d;--bs-dropdown-header-padding-x: 1rem;--bs-dropdown-header-padding-y: 0.5rem;position:absolute;z-index:var(--bs-dropdown-zindex);display:none;min-width:var(--bs-dropdown-min-width);padding:var(--bs-dropdown-padding-y) var(--bs-dropdown-padding-x);margin:0;font-size:var(--bs-dropdown-font-size);color:var(--bs-dropdown-color);text-align:left;list-style:none;background-color:var(--bs-dropdown-bg);background-clip:padding-box;border:var(--bs-dropdown-border-width) solid var(--bs-dropdown-border-color)}.dropdown-menu[data-bs-popper]{top:100%;left:0;margin-top:var(--bs-dropdown-spacer)}.dropdown-menu-start{--bs-position: start}.dropdown-menu-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-end{--bs-position: end}.dropdown-menu-end[data-bs-popper]{right:0;left:auto}@media(min-width: 576px){.dropdown-menu-sm-start{--bs-position: start}.dropdown-menu-sm-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-sm-end{--bs-position: end}.dropdown-menu-sm-end[data-bs-popper]{right:0;left:auto}}@media(min-width: 768px){.dropdown-menu-md-start{--bs-position: start}.dropdown-menu-md-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-md-end{--bs-position: end}.dropdown-menu-md-end[data-bs-popper]{right:0;left:auto}}@media(min-width: 992px){.dropdown-menu-lg-start{--bs-position: start}.dropdown-menu-lg-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-lg-end{--bs-position: end}.dropdown-menu-lg-end[data-bs-popper]{right:0;left:auto}}@media(min-width: 1200px){.dropdown-menu-xl-start{--bs-position: start}.dropdown-menu-xl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xl-end{--bs-position: end}.dropdown-menu-xl-end[data-bs-popper]{right:0;left:auto}}@media(min-width: 1400px){.dropdown-menu-xxl-start{--bs-position: start}.dropdown-menu-xxl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xxl-end{--bs-position: end}.dropdown-menu-xxl-end[data-bs-popper]{right:0;left:auto}}.dropup .dropdown-menu[data-bs-popper]{top:auto;bottom:100%;margin-top:0;margin-bottom:var(--bs-dropdown-spacer)}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid rgba(0,0,0,0);border-bottom:.3em solid;border-left:.3em solid rgba(0,0,0,0)}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropend .dropdown-menu[data-bs-popper]{top:0;right:auto;left:100%;margin-top:0;margin-left:var(--bs-dropdown-spacer)}.dropend .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid rgba(0,0,0,0);border-right:0;border-bottom:.3em solid rgba(0,0,0,0);border-left:.3em solid}.dropend .dropdown-toggle:empty::after{margin-left:0}.dropend .dropdown-toggle::after{vertical-align:0}.dropstart .dropdown-menu[data-bs-popper]{top:0;right:100%;left:auto;margin-top:0;margin-right:var(--bs-dropdown-spacer)}.dropstart .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropstart .dropdown-toggle::after{display:none}.dropstart .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid rgba(0,0,0,0);border-right:.3em solid;border-bottom:.3em solid rgba(0,0,0,0)}.dropstart .dropdown-toggle:empty::after{margin-left:0}.dropstart .dropdown-toggle::before{vertical-align:0}.dropdown-divider{height:0;margin:var(--bs-dropdown-divider-margin-y) 0;overflow:hidden;border-top:1px solid var(--bs-dropdown-divider-bg);opacity:1}.dropdown-item{display:block;width:100%;padding:var(--bs-dropdown-item-padding-y) var(--bs-dropdown-item-padding-x);clear:both;font-weight:400;color:var(--bs-dropdown-link-color);text-align:inherit;text-decoration:none;-webkit-text-decoration:none;-moz-text-decoration:none;-ms-text-decoration:none;-o-text-decoration:none;white-space:nowrap;background-color:rgba(0,0,0,0);border:0}.dropdown-item:hover,.dropdown-item:focus{color:var(--bs-dropdown-link-hover-color);background-color:var(--bs-dropdown-link-hover-bg)}.dropdown-item.active,.dropdown-item:active{color:var(--bs-dropdown-link-active-color);text-decoration:none;background-color:var(--bs-dropdown-link-active-bg)}.dropdown-item.disabled,.dropdown-item:disabled{color:var(--bs-dropdown-link-disabled-color);pointer-events:none;background-color:rgba(0,0,0,0)}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:var(--bs-dropdown-header-padding-y) var(--bs-dropdown-header-padding-x);margin-bottom:0;font-size:0.875rem;color:var(--bs-dropdown-header-color);white-space:nowrap}.dropdown-item-text{display:block;padding:var(--bs-dropdown-item-padding-y) var(--bs-dropdown-item-padding-x);color:var(--bs-dropdown-link-color)}.dropdown-menu-dark{--bs-dropdown-color: #dee2e6;--bs-dropdown-bg: #343a40;--bs-dropdown-border-color: rgba(0, 0, 0, 0.175);--bs-dropdown-box-shadow: ;--bs-dropdown-link-color: #dee2e6;--bs-dropdown-link-hover-color: #fff;--bs-dropdown-divider-bg: rgba(0, 0, 0, 0.175);--bs-dropdown-link-hover-bg: rgba(255, 255, 255, 0.15);--bs-dropdown-link-active-color: #fff;--bs-dropdown-link-active-bg: #2780e3;--bs-dropdown-link-disabled-color: #adb5bd;--bs-dropdown-header-color: #adb5bd}.btn-group,.btn-group-vertical{position:relative;display:inline-flex;vertical-align:middle}.btn-group>.btn,.btn-group-vertical>.btn{position:relative;flex:1 1 auto;-webkit-flex:1 1 auto}.btn-group>.btn-check:checked+.btn,.btn-group>.btn-check:focus+.btn,.btn-group>.btn:hover,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active,.btn-group-vertical>.btn-check:checked+.btn,.btn-group-vertical>.btn-check:focus+.btn,.btn-group-vertical>.btn:hover,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn.active{z-index:1}.btn-toolbar{display:flex;display:-webkit-flex;flex-wrap:wrap;-webkit-flex-wrap:wrap;justify-content:flex-start;-webkit-justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>:not(.btn-check:first-child)+.btn,.btn-group>.btn-group:not(:first-child){margin-left:calc(1px*-1)}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after,.dropend .dropdown-toggle-split::after{margin-left:0}.dropstart .dropdown-toggle-split::before{margin-right:0}.btn-sm+.dropdown-toggle-split,.btn-group-sm>.btn+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-lg+.dropdown-toggle-split,.btn-group-lg>.btn+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{flex-direction:column;-webkit-flex-direction:column;align-items:flex-start;-webkit-align-items:flex-start;justify-content:center;-webkit-justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn:not(:first-child),.btn-group-vertical>.btn-group:not(:first-child){margin-top:calc(1px*-1)}.nav{--bs-nav-link-padding-x: 1rem;--bs-nav-link-padding-y: 0.5rem;--bs-nav-link-font-weight: ;--bs-nav-link-color: #2761e3;--bs-nav-link-hover-color: #1f4eb6;--bs-nav-link-disabled-color: rgba(52, 58, 64, 0.75);display:flex;display:-webkit-flex;flex-wrap:wrap;-webkit-flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:var(--bs-nav-link-padding-y) var(--bs-nav-link-padding-x);font-size:var(--bs-nav-link-font-size);font-weight:var(--bs-nav-link-font-weight);color:var(--bs-nav-link-color);text-decoration:none;-webkit-text-decoration:none;-moz-text-decoration:none;-ms-text-decoration:none;-o-text-decoration:none;background:none;border:0;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out}@media(prefers-reduced-motion: reduce){.nav-link{transition:none}}.nav-link:hover,.nav-link:focus{color:var(--bs-nav-link-hover-color)}.nav-link:focus-visible{outline:0;box-shadow:0 0 0 .25rem rgba(39,128,227,.25)}.nav-link.disabled,.nav-link:disabled{color:var(--bs-nav-link-disabled-color);pointer-events:none;cursor:default}.nav-tabs{--bs-nav-tabs-border-width: 1px;--bs-nav-tabs-border-color: #dee2e6;--bs-nav-tabs-border-radius: 0.25rem;--bs-nav-tabs-link-hover-border-color: #e9ecef #e9ecef #dee2e6;--bs-nav-tabs-link-active-color: #000;--bs-nav-tabs-link-active-bg: #fff;--bs-nav-tabs-link-active-border-color: #dee2e6 #dee2e6 #fff;border-bottom:var(--bs-nav-tabs-border-width) solid var(--bs-nav-tabs-border-color)}.nav-tabs .nav-link{margin-bottom:calc(-1*var(--bs-nav-tabs-border-width));border:var(--bs-nav-tabs-border-width) solid rgba(0,0,0,0)}.nav-tabs .nav-link:hover,.nav-tabs .nav-link:focus{isolation:isolate;border-color:var(--bs-nav-tabs-link-hover-border-color)}.nav-tabs .nav-link.active,.nav-tabs .nav-item.show .nav-link{color:var(--bs-nav-tabs-link-active-color);background-color:var(--bs-nav-tabs-link-active-bg);border-color:var(--bs-nav-tabs-link-active-border-color)}.nav-tabs .dropdown-menu{margin-top:calc(-1*var(--bs-nav-tabs-border-width))}.nav-pills{--bs-nav-pills-border-radius: 0.25rem;--bs-nav-pills-link-active-color: #fff;--bs-nav-pills-link-active-bg: #2780e3}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:var(--bs-nav-pills-link-active-color);background-color:var(--bs-nav-pills-link-active-bg)}.nav-underline{--bs-nav-underline-gap: 1rem;--bs-nav-underline-border-width: 0.125rem;--bs-nav-underline-link-active-color: #000;gap:var(--bs-nav-underline-gap)}.nav-underline .nav-link{padding-right:0;padding-left:0;border-bottom:var(--bs-nav-underline-border-width) solid rgba(0,0,0,0)}.nav-underline .nav-link:hover,.nav-underline .nav-link:focus{border-bottom-color:currentcolor}.nav-underline .nav-link.active,.nav-underline .show>.nav-link{font-weight:700;color:var(--bs-nav-underline-link-active-color);border-bottom-color:currentcolor}.nav-fill>.nav-link,.nav-fill .nav-item{flex:1 1 auto;-webkit-flex:1 1 auto;text-align:center}.nav-justified>.nav-link,.nav-justified .nav-item{flex-basis:0;-webkit-flex-basis:0;flex-grow:1;-webkit-flex-grow:1;text-align:center}.nav-fill .nav-item .nav-link,.nav-justified .nav-item .nav-link{width:100%}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{--bs-navbar-padding-x: 0;--bs-navbar-padding-y: 0.5rem;--bs-navbar-color: #fefdfd;--bs-navbar-hover-color: rgba(253, 253, 255, 0.8);--bs-navbar-disabled-color: rgba(254, 253, 253, 0.75);--bs-navbar-active-color: #fdfdff;--bs-navbar-brand-padding-y: 0.3125rem;--bs-navbar-brand-margin-end: 1rem;--bs-navbar-brand-font-size: 1.25rem;--bs-navbar-brand-color: #fefdfd;--bs-navbar-brand-hover-color: #fdfdff;--bs-navbar-nav-link-padding-x: 0.5rem;--bs-navbar-toggler-padding-y: 0.25;--bs-navbar-toggler-padding-x: 0;--bs-navbar-toggler-font-size: 1.25rem;--bs-navbar-toggler-icon-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='%23fefdfd' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e");--bs-navbar-toggler-border-color: rgba(254, 253, 253, 0);--bs-navbar-toggler-border-radius: 0.25rem;--bs-navbar-toggler-focus-width: 0.25rem;--bs-navbar-toggler-transition: box-shadow 0.15s ease-in-out;position:relative;display:flex;display:-webkit-flex;flex-wrap:wrap;-webkit-flex-wrap:wrap;align-items:center;-webkit-align-items:center;justify-content:space-between;-webkit-justify-content:space-between;padding:var(--bs-navbar-padding-y) var(--bs-navbar-padding-x)}.navbar>.container,.navbar>.container-fluid,.navbar>.container-sm,.navbar>.container-md,.navbar>.container-lg,.navbar>.container-xl,.navbar>.container-xxl{display:flex;display:-webkit-flex;flex-wrap:inherit;-webkit-flex-wrap:inherit;align-items:center;-webkit-align-items:center;justify-content:space-between;-webkit-justify-content:space-between}.navbar-brand{padding-top:var(--bs-navbar-brand-padding-y);padding-bottom:var(--bs-navbar-brand-padding-y);margin-right:var(--bs-navbar-brand-margin-end);font-size:var(--bs-navbar-brand-font-size);color:var(--bs-navbar-brand-color);text-decoration:none;-webkit-text-decoration:none;-moz-text-decoration:none;-ms-text-decoration:none;-o-text-decoration:none;white-space:nowrap}.navbar-brand:hover,.navbar-brand:focus{color:var(--bs-navbar-brand-hover-color)}.navbar-nav{--bs-nav-link-padding-x: 0;--bs-nav-link-padding-y: 0.5rem;--bs-nav-link-font-weight: ;--bs-nav-link-color: var(--bs-navbar-color);--bs-nav-link-hover-color: var(--bs-navbar-hover-color);--bs-nav-link-disabled-color: var(--bs-navbar-disabled-color);display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link.active,.navbar-nav .nav-link.show{color:var(--bs-navbar-active-color)}.navbar-nav .dropdown-menu{position:static}.navbar-text{padding-top:.5rem;padding-bottom:.5rem;color:var(--bs-navbar-color)}.navbar-text a,.navbar-text a:hover,.navbar-text a:focus{color:var(--bs-navbar-active-color)}.navbar-collapse{flex-basis:100%;-webkit-flex-basis:100%;flex-grow:1;-webkit-flex-grow:1;align-items:center;-webkit-align-items:center}.navbar-toggler{padding:var(--bs-navbar-toggler-padding-y) var(--bs-navbar-toggler-padding-x);font-size:var(--bs-navbar-toggler-font-size);line-height:1;color:var(--bs-navbar-color);background-color:rgba(0,0,0,0);border:var(--bs-border-width) solid var(--bs-navbar-toggler-border-color);transition:var(--bs-navbar-toggler-transition)}@media(prefers-reduced-motion: reduce){.navbar-toggler{transition:none}}.navbar-toggler:hover{text-decoration:none}.navbar-toggler:focus{text-decoration:none;outline:0;box-shadow:0 0 0 var(--bs-navbar-toggler-focus-width)}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;background-image:var(--bs-navbar-toggler-icon-bg);background-repeat:no-repeat;background-position:center;background-size:100%}.navbar-nav-scroll{max-height:var(--bs-scroll-height, 75vh);overflow-y:auto}@media(min-width: 576px){.navbar-expand-sm{flex-wrap:nowrap;-webkit-flex-wrap:nowrap;justify-content:flex-start;-webkit-justify-content:flex-start}.navbar-expand-sm .navbar-nav{flex-direction:row;-webkit-flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-sm .navbar-nav-scroll{overflow:visible}.navbar-expand-sm .navbar-collapse{display:flex !important;display:-webkit-flex !important;flex-basis:auto;-webkit-flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}.navbar-expand-sm .offcanvas{position:static;z-index:auto;flex-grow:1;-webkit-flex-grow:1;width:auto !important;height:auto !important;visibility:visible !important;background-color:rgba(0,0,0,0) !important;border:0 !important;transform:none !important;transition:none}.navbar-expand-sm .offcanvas .offcanvas-header{display:none}.navbar-expand-sm .offcanvas .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible}}@media(min-width: 768px){.navbar-expand-md{flex-wrap:nowrap;-webkit-flex-wrap:nowrap;justify-content:flex-start;-webkit-justify-content:flex-start}.navbar-expand-md .navbar-nav{flex-direction:row;-webkit-flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-md .navbar-nav-scroll{overflow:visible}.navbar-expand-md .navbar-collapse{display:flex !important;display:-webkit-flex !important;flex-basis:auto;-webkit-flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}.navbar-expand-md .offcanvas{position:static;z-index:auto;flex-grow:1;-webkit-flex-grow:1;width:auto !important;height:auto !important;visibility:visible !important;background-color:rgba(0,0,0,0) !important;border:0 !important;transform:none !important;transition:none}.navbar-expand-md .offcanvas .offcanvas-header{display:none}.navbar-expand-md .offcanvas .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible}}@media(min-width: 992px){.navbar-expand-lg{flex-wrap:nowrap;-webkit-flex-wrap:nowrap;justify-content:flex-start;-webkit-justify-content:flex-start}.navbar-expand-lg .navbar-nav{flex-direction:row;-webkit-flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-lg .navbar-nav-scroll{overflow:visible}.navbar-expand-lg .navbar-collapse{display:flex !important;display:-webkit-flex !important;flex-basis:auto;-webkit-flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}.navbar-expand-lg .offcanvas{position:static;z-index:auto;flex-grow:1;-webkit-flex-grow:1;width:auto !important;height:auto !important;visibility:visible !important;background-color:rgba(0,0,0,0) !important;border:0 !important;transform:none !important;transition:none}.navbar-expand-lg .offcanvas .offcanvas-header{display:none}.navbar-expand-lg .offcanvas .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible}}@media(min-width: 1200px){.navbar-expand-xl{flex-wrap:nowrap;-webkit-flex-wrap:nowrap;justify-content:flex-start;-webkit-justify-content:flex-start}.navbar-expand-xl .navbar-nav{flex-direction:row;-webkit-flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-xl .navbar-nav-scroll{overflow:visible}.navbar-expand-xl .navbar-collapse{display:flex !important;display:-webkit-flex !important;flex-basis:auto;-webkit-flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}.navbar-expand-xl .offcanvas{position:static;z-index:auto;flex-grow:1;-webkit-flex-grow:1;width:auto !important;height:auto !important;visibility:visible !important;background-color:rgba(0,0,0,0) !important;border:0 !important;transform:none !important;transition:none}.navbar-expand-xl .offcanvas .offcanvas-header{display:none}.navbar-expand-xl .offcanvas .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible}}@media(min-width: 1400px){.navbar-expand-xxl{flex-wrap:nowrap;-webkit-flex-wrap:nowrap;justify-content:flex-start;-webkit-justify-content:flex-start}.navbar-expand-xxl .navbar-nav{flex-direction:row;-webkit-flex-direction:row}.navbar-expand-xxl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xxl .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand-xxl .navbar-nav-scroll{overflow:visible}.navbar-expand-xxl .navbar-collapse{display:flex !important;display:-webkit-flex !important;flex-basis:auto;-webkit-flex-basis:auto}.navbar-expand-xxl .navbar-toggler{display:none}.navbar-expand-xxl .offcanvas{position:static;z-index:auto;flex-grow:1;-webkit-flex-grow:1;width:auto !important;height:auto !important;visibility:visible !important;background-color:rgba(0,0,0,0) !important;border:0 !important;transform:none !important;transition:none}.navbar-expand-xxl .offcanvas .offcanvas-header{display:none}.navbar-expand-xxl .offcanvas .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible}}.navbar-expand{flex-wrap:nowrap;-webkit-flex-wrap:nowrap;justify-content:flex-start;-webkit-justify-content:flex-start}.navbar-expand .navbar-nav{flex-direction:row;-webkit-flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:var(--bs-navbar-nav-link-padding-x);padding-left:var(--bs-navbar-nav-link-padding-x)}.navbar-expand .navbar-nav-scroll{overflow:visible}.navbar-expand .navbar-collapse{display:flex !important;display:-webkit-flex !important;flex-basis:auto;-webkit-flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-expand .offcanvas{position:static;z-index:auto;flex-grow:1;-webkit-flex-grow:1;width:auto !important;height:auto !important;visibility:visible !important;background-color:rgba(0,0,0,0) !important;border:0 !important;transform:none !important;transition:none}.navbar-expand .offcanvas .offcanvas-header{display:none}.navbar-expand .offcanvas .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible}.navbar-dark,.navbar[data-bs-theme=dark]{--bs-navbar-color: #fefdfd;--bs-navbar-hover-color: rgba(253, 253, 255, 0.8);--bs-navbar-disabled-color: rgba(254, 253, 253, 0.75);--bs-navbar-active-color: #fdfdff;--bs-navbar-brand-color: #fefdfd;--bs-navbar-brand-hover-color: #fdfdff;--bs-navbar-toggler-border-color: rgba(254, 253, 253, 0);--bs-navbar-toggler-icon-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='%23fefdfd' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}[data-bs-theme=dark] .navbar-toggler-icon{--bs-navbar-toggler-icon-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='%23fefdfd' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.card{--bs-card-spacer-y: 1rem;--bs-card-spacer-x: 1rem;--bs-card-title-spacer-y: 0.5rem;--bs-card-title-color: ;--bs-card-subtitle-color: ;--bs-card-border-width: 1px;--bs-card-border-color: rgba(0, 0, 0, 0.175);--bs-card-border-radius: 0.25rem;--bs-card-box-shadow: ;--bs-card-inner-border-radius: calc(0.25rem - 1px);--bs-card-cap-padding-y: 0.5rem;--bs-card-cap-padding-x: 1rem;--bs-card-cap-bg: rgba(52, 58, 64, 0.25);--bs-card-cap-color: ;--bs-card-height: ;--bs-card-color: ;--bs-card-bg: #fff;--bs-card-img-overlay-padding: 1rem;--bs-card-group-margin: 0.75rem;position:relative;display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;min-width:0;height:var(--bs-card-height);color:var(--bs-body-color);word-wrap:break-word;background-color:var(--bs-card-bg);background-clip:border-box;border:var(--bs-card-border-width) solid var(--bs-card-border-color)}.card>hr{margin-right:0;margin-left:0}.card>.list-group{border-top:inherit;border-bottom:inherit}.card>.list-group:first-child{border-top-width:0}.card>.list-group:last-child{border-bottom-width:0}.card>.card-header+.list-group,.card>.list-group+.card-footer{border-top:0}.card-body{flex:1 1 auto;-webkit-flex:1 1 auto;padding:var(--bs-card-spacer-y) var(--bs-card-spacer-x);color:var(--bs-card-color)}.card-title{margin-bottom:var(--bs-card-title-spacer-y);color:var(--bs-card-title-color)}.card-subtitle{margin-top:calc(-0.5*var(--bs-card-title-spacer-y));margin-bottom:0;color:var(--bs-card-subtitle-color)}.card-text:last-child{margin-bottom:0}.card-link+.card-link{margin-left:var(--bs-card-spacer-x)}.card-header{padding:var(--bs-card-cap-padding-y) var(--bs-card-cap-padding-x);margin-bottom:0;color:var(--bs-card-cap-color);background-color:var(--bs-card-cap-bg);border-bottom:var(--bs-card-border-width) solid var(--bs-card-border-color)}.card-footer{padding:var(--bs-card-cap-padding-y) var(--bs-card-cap-padding-x);color:var(--bs-card-cap-color);background-color:var(--bs-card-cap-bg);border-top:var(--bs-card-border-width) solid var(--bs-card-border-color)}.card-header-tabs{margin-right:calc(-0.5*var(--bs-card-cap-padding-x));margin-bottom:calc(-1*var(--bs-card-cap-padding-y));margin-left:calc(-0.5*var(--bs-card-cap-padding-x));border-bottom:0}.card-header-tabs .nav-link.active{background-color:var(--bs-card-bg);border-bottom-color:var(--bs-card-bg)}.card-header-pills{margin-right:calc(-0.5*var(--bs-card-cap-padding-x));margin-left:calc(-0.5*var(--bs-card-cap-padding-x))}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:var(--bs-card-img-overlay-padding)}.card-img,.card-img-top,.card-img-bottom{width:100%}.card-group>.card{margin-bottom:var(--bs-card-group-margin)}@media(min-width: 576px){.card-group{display:flex;display:-webkit-flex;flex-flow:row wrap;-webkit-flex-flow:row wrap}.card-group>.card{flex:1 0 0%;-webkit-flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}}.accordion{--bs-accordion-color: #343a40;--bs-accordion-bg: #fff;--bs-accordion-transition: color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out, border-radius 0.15s ease;--bs-accordion-border-color: #dee2e6;--bs-accordion-border-width: 1px;--bs-accordion-border-radius: 0.25rem;--bs-accordion-inner-border-radius: calc(0.25rem - 1px);--bs-accordion-btn-padding-x: 1.25rem;--bs-accordion-btn-padding-y: 1rem;--bs-accordion-btn-color: #343a40;--bs-accordion-btn-bg: #fff;--bs-accordion-btn-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23343a40'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");--bs-accordion-btn-icon-width: 1.25rem;--bs-accordion-btn-icon-transform: rotate(-180deg);--bs-accordion-btn-icon-transition: transform 0.2s ease-in-out;--bs-accordion-btn-active-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%2310335b'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");--bs-accordion-btn-focus-border-color: #93c0f1;--bs-accordion-btn-focus-box-shadow: 0 0 0 0.25rem rgba(39, 128, 227, 0.25);--bs-accordion-body-padding-x: 1.25rem;--bs-accordion-body-padding-y: 1rem;--bs-accordion-active-color: #10335b;--bs-accordion-active-bg: #d4e6f9}.accordion-button{position:relative;display:flex;display:-webkit-flex;align-items:center;-webkit-align-items:center;width:100%;padding:var(--bs-accordion-btn-padding-y) var(--bs-accordion-btn-padding-x);font-size:1rem;color:var(--bs-accordion-btn-color);text-align:left;background-color:var(--bs-accordion-btn-bg);border:0;overflow-anchor:none;transition:var(--bs-accordion-transition)}@media(prefers-reduced-motion: reduce){.accordion-button{transition:none}}.accordion-button:not(.collapsed){color:var(--bs-accordion-active-color);background-color:var(--bs-accordion-active-bg);box-shadow:inset 0 calc(-1*var(--bs-accordion-border-width)) 0 var(--bs-accordion-border-color)}.accordion-button:not(.collapsed)::after{background-image:var(--bs-accordion-btn-active-icon);transform:var(--bs-accordion-btn-icon-transform)}.accordion-button::after{flex-shrink:0;-webkit-flex-shrink:0;width:var(--bs-accordion-btn-icon-width);height:var(--bs-accordion-btn-icon-width);margin-left:auto;content:"";background-image:var(--bs-accordion-btn-icon);background-repeat:no-repeat;background-size:var(--bs-accordion-btn-icon-width);transition:var(--bs-accordion-btn-icon-transition)}@media(prefers-reduced-motion: reduce){.accordion-button::after{transition:none}}.accordion-button:hover{z-index:2}.accordion-button:focus{z-index:3;border-color:var(--bs-accordion-btn-focus-border-color);outline:0;box-shadow:var(--bs-accordion-btn-focus-box-shadow)}.accordion-header{margin-bottom:0}.accordion-item{color:var(--bs-accordion-color);background-color:var(--bs-accordion-bg);border:var(--bs-accordion-border-width) solid var(--bs-accordion-border-color)}.accordion-item:not(:first-of-type){border-top:0}.accordion-body{padding:var(--bs-accordion-body-padding-y) var(--bs-accordion-body-padding-x)}.accordion-flush .accordion-collapse{border-width:0}.accordion-flush .accordion-item{border-right:0;border-left:0}.accordion-flush .accordion-item:first-child{border-top:0}.accordion-flush .accordion-item:last-child{border-bottom:0}[data-bs-theme=dark] .accordion-button::after{--bs-accordion-btn-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%237db3ee'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");--bs-accordion-btn-active-icon: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%237db3ee'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e")}.breadcrumb{--bs-breadcrumb-padding-x: 0;--bs-breadcrumb-padding-y: 0;--bs-breadcrumb-margin-bottom: 1rem;--bs-breadcrumb-bg: ;--bs-breadcrumb-border-radius: ;--bs-breadcrumb-divider-color: rgba(52, 58, 64, 0.75);--bs-breadcrumb-item-padding-x: 0.5rem;--bs-breadcrumb-item-active-color: rgba(52, 58, 64, 0.75);display:flex;display:-webkit-flex;flex-wrap:wrap;-webkit-flex-wrap:wrap;padding:var(--bs-breadcrumb-padding-y) var(--bs-breadcrumb-padding-x);margin-bottom:var(--bs-breadcrumb-margin-bottom);font-size:var(--bs-breadcrumb-font-size);list-style:none;background-color:var(--bs-breadcrumb-bg)}.breadcrumb-item+.breadcrumb-item{padding-left:var(--bs-breadcrumb-item-padding-x)}.breadcrumb-item+.breadcrumb-item::before{float:left;padding-right:var(--bs-breadcrumb-item-padding-x);color:var(--bs-breadcrumb-divider-color);content:var(--bs-breadcrumb-divider, ">") /* rtl: var(--bs-breadcrumb-divider, ">") */}.breadcrumb-item.active{color:var(--bs-breadcrumb-item-active-color)}.pagination{--bs-pagination-padding-x: 0.75rem;--bs-pagination-padding-y: 0.375rem;--bs-pagination-font-size:1rem;--bs-pagination-color: #2761e3;--bs-pagination-bg: #fff;--bs-pagination-border-width: 1px;--bs-pagination-border-color: #dee2e6;--bs-pagination-border-radius: 0.25rem;--bs-pagination-hover-color: #1f4eb6;--bs-pagination-hover-bg: #f8f9fa;--bs-pagination-hover-border-color: #dee2e6;--bs-pagination-focus-color: #1f4eb6;--bs-pagination-focus-bg: #e9ecef;--bs-pagination-focus-box-shadow: 0 0 0 0.25rem rgba(39, 128, 227, 0.25);--bs-pagination-active-color: #fff;--bs-pagination-active-bg: #2780e3;--bs-pagination-active-border-color: #2780e3;--bs-pagination-disabled-color: rgba(52, 58, 64, 0.75);--bs-pagination-disabled-bg: #e9ecef;--bs-pagination-disabled-border-color: #dee2e6;display:flex;display:-webkit-flex;padding-left:0;list-style:none}.page-link{position:relative;display:block;padding:var(--bs-pagination-padding-y) var(--bs-pagination-padding-x);font-size:var(--bs-pagination-font-size);color:var(--bs-pagination-color);text-decoration:none;-webkit-text-decoration:none;-moz-text-decoration:none;-ms-text-decoration:none;-o-text-decoration:none;background-color:var(--bs-pagination-bg);border:var(--bs-pagination-border-width) solid var(--bs-pagination-border-color);transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media(prefers-reduced-motion: reduce){.page-link{transition:none}}.page-link:hover{z-index:2;color:var(--bs-pagination-hover-color);background-color:var(--bs-pagination-hover-bg);border-color:var(--bs-pagination-hover-border-color)}.page-link:focus{z-index:3;color:var(--bs-pagination-focus-color);background-color:var(--bs-pagination-focus-bg);outline:0;box-shadow:var(--bs-pagination-focus-box-shadow)}.page-link.active,.active>.page-link{z-index:3;color:var(--bs-pagination-active-color);background-color:var(--bs-pagination-active-bg);border-color:var(--bs-pagination-active-border-color)}.page-link.disabled,.disabled>.page-link{color:var(--bs-pagination-disabled-color);pointer-events:none;background-color:var(--bs-pagination-disabled-bg);border-color:var(--bs-pagination-disabled-border-color)}.page-item:not(:first-child) .page-link{margin-left:calc(1px*-1)}.pagination-lg{--bs-pagination-padding-x: 1.5rem;--bs-pagination-padding-y: 0.75rem;--bs-pagination-font-size:1.25rem;--bs-pagination-border-radius: 0.5rem}.pagination-sm{--bs-pagination-padding-x: 0.5rem;--bs-pagination-padding-y: 0.25rem;--bs-pagination-font-size:0.875rem;--bs-pagination-border-radius: 0.2em}.badge{--bs-badge-padding-x: 0.65em;--bs-badge-padding-y: 0.35em;--bs-badge-font-size:0.75em;--bs-badge-font-weight: 700;--bs-badge-color: #fff;--bs-badge-border-radius: 0.25rem;display:inline-block;padding:var(--bs-badge-padding-y) var(--bs-badge-padding-x);font-size:var(--bs-badge-font-size);font-weight:var(--bs-badge-font-weight);line-height:1;color:var(--bs-badge-color);text-align:center;white-space:nowrap;vertical-align:baseline}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.alert{--bs-alert-bg: transparent;--bs-alert-padding-x: 1rem;--bs-alert-padding-y: 1rem;--bs-alert-margin-bottom: 1rem;--bs-alert-color: inherit;--bs-alert-border-color: transparent;--bs-alert-border: 0 solid var(--bs-alert-border-color);--bs-alert-border-radius: 0.25rem;--bs-alert-link-color: inherit;position:relative;padding:var(--bs-alert-padding-y) var(--bs-alert-padding-x);margin-bottom:var(--bs-alert-margin-bottom);color:var(--bs-alert-color);background-color:var(--bs-alert-bg);border:var(--bs-alert-border)}.alert-heading{color:inherit}.alert-link{font-weight:700;color:var(--bs-alert-link-color)}.alert-dismissible{padding-right:3rem}.alert-dismissible .btn-close{position:absolute;top:0;right:0;z-index:2;padding:1.25rem 1rem}.alert-default{--bs-alert-color: var(--bs-default-text-emphasis);--bs-alert-bg: var(--bs-default-bg-subtle);--bs-alert-border-color: var(--bs-default-border-subtle);--bs-alert-link-color: var(--bs-default-text-emphasis)}.alert-primary{--bs-alert-color: var(--bs-primary-text-emphasis);--bs-alert-bg: var(--bs-primary-bg-subtle);--bs-alert-border-color: var(--bs-primary-border-subtle);--bs-alert-link-color: var(--bs-primary-text-emphasis)}.alert-secondary{--bs-alert-color: var(--bs-secondary-text-emphasis);--bs-alert-bg: var(--bs-secondary-bg-subtle);--bs-alert-border-color: var(--bs-secondary-border-subtle);--bs-alert-link-color: var(--bs-secondary-text-emphasis)}.alert-success{--bs-alert-color: var(--bs-success-text-emphasis);--bs-alert-bg: var(--bs-success-bg-subtle);--bs-alert-border-color: var(--bs-success-border-subtle);--bs-alert-link-color: var(--bs-success-text-emphasis)}.alert-info{--bs-alert-color: var(--bs-info-text-emphasis);--bs-alert-bg: var(--bs-info-bg-subtle);--bs-alert-border-color: var(--bs-info-border-subtle);--bs-alert-link-color: var(--bs-info-text-emphasis)}.alert-warning{--bs-alert-color: var(--bs-warning-text-emphasis);--bs-alert-bg: var(--bs-warning-bg-subtle);--bs-alert-border-color: var(--bs-warning-border-subtle);--bs-alert-link-color: var(--bs-warning-text-emphasis)}.alert-danger{--bs-alert-color: var(--bs-danger-text-emphasis);--bs-alert-bg: var(--bs-danger-bg-subtle);--bs-alert-border-color: var(--bs-danger-border-subtle);--bs-alert-link-color: var(--bs-danger-text-emphasis)}.alert-light{--bs-alert-color: var(--bs-light-text-emphasis);--bs-alert-bg: var(--bs-light-bg-subtle);--bs-alert-border-color: var(--bs-light-border-subtle);--bs-alert-link-color: var(--bs-light-text-emphasis)}.alert-dark{--bs-alert-color: var(--bs-dark-text-emphasis);--bs-alert-bg: var(--bs-dark-bg-subtle);--bs-alert-border-color: var(--bs-dark-border-subtle);--bs-alert-link-color: var(--bs-dark-text-emphasis)}@keyframes progress-bar-stripes{0%{background-position-x:.5rem}}.progress,.progress-stacked{--bs-progress-height: 0.5rem;--bs-progress-font-size:0.75rem;--bs-progress-bg: #e9ecef;--bs-progress-border-radius: 0.25rem;--bs-progress-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.075);--bs-progress-bar-color: #fff;--bs-progress-bar-bg: #2780e3;--bs-progress-bar-transition: width 0.6s ease;display:flex;display:-webkit-flex;height:var(--bs-progress-height);overflow:hidden;font-size:var(--bs-progress-font-size);background-color:var(--bs-progress-bg)}.progress-bar{display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;justify-content:center;-webkit-justify-content:center;overflow:hidden;color:var(--bs-progress-bar-color);text-align:center;white-space:nowrap;background-color:var(--bs-progress-bar-bg);transition:var(--bs-progress-bar-transition)}@media(prefers-reduced-motion: reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);background-size:var(--bs-progress-height) var(--bs-progress-height)}.progress-stacked>.progress{overflow:visible}.progress-stacked>.progress>.progress-bar{width:100%}.progress-bar-animated{animation:1s linear infinite progress-bar-stripes}@media(prefers-reduced-motion: reduce){.progress-bar-animated{animation:none}}.list-group{--bs-list-group-color: #343a40;--bs-list-group-bg: #fff;--bs-list-group-border-color: #dee2e6;--bs-list-group-border-width: 1px;--bs-list-group-border-radius: 0.25rem;--bs-list-group-item-padding-x: 1rem;--bs-list-group-item-padding-y: 0.5rem;--bs-list-group-action-color: rgba(52, 58, 64, 0.75);--bs-list-group-action-hover-color: #000;--bs-list-group-action-hover-bg: #f8f9fa;--bs-list-group-action-active-color: #343a40;--bs-list-group-action-active-bg: #e9ecef;--bs-list-group-disabled-color: rgba(52, 58, 64, 0.75);--bs-list-group-disabled-bg: #fff;--bs-list-group-active-color: #fff;--bs-list-group-active-bg: #2780e3;--bs-list-group-active-border-color: #2780e3;display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;padding-left:0;margin-bottom:0}.list-group-numbered{list-style-type:none;counter-reset:section}.list-group-numbered>.list-group-item::before{content:counters(section, ".") ". ";counter-increment:section}.list-group-item-action{width:100%;color:var(--bs-list-group-action-color);text-align:inherit}.list-group-item-action:hover,.list-group-item-action:focus{z-index:1;color:var(--bs-list-group-action-hover-color);text-decoration:none;background-color:var(--bs-list-group-action-hover-bg)}.list-group-item-action:active{color:var(--bs-list-group-action-active-color);background-color:var(--bs-list-group-action-active-bg)}.list-group-item{position:relative;display:block;padding:var(--bs-list-group-item-padding-y) var(--bs-list-group-item-padding-x);color:var(--bs-list-group-color);text-decoration:none;-webkit-text-decoration:none;-moz-text-decoration:none;-ms-text-decoration:none;-o-text-decoration:none;background-color:var(--bs-list-group-bg);border:var(--bs-list-group-border-width) solid var(--bs-list-group-border-color)}.list-group-item.disabled,.list-group-item:disabled{color:var(--bs-list-group-disabled-color);pointer-events:none;background-color:var(--bs-list-group-disabled-bg)}.list-group-item.active{z-index:2;color:var(--bs-list-group-active-color);background-color:var(--bs-list-group-active-bg);border-color:var(--bs-list-group-active-border-color)}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:calc(-1*var(--bs-list-group-border-width));border-top-width:var(--bs-list-group-border-width)}.list-group-horizontal{flex-direction:row;-webkit-flex-direction:row}.list-group-horizontal>.list-group-item.active{margin-top:0}.list-group-horizontal>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal>.list-group-item+.list-group-item.active{margin-left:calc(-1*var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}@media(min-width: 576px){.list-group-horizontal-sm{flex-direction:row;-webkit-flex-direction:row}.list-group-horizontal-sm>.list-group-item.active{margin-top:0}.list-group-horizontal-sm>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-sm>.list-group-item+.list-group-item.active{margin-left:calc(-1*var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media(min-width: 768px){.list-group-horizontal-md{flex-direction:row;-webkit-flex-direction:row}.list-group-horizontal-md>.list-group-item.active{margin-top:0}.list-group-horizontal-md>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-md>.list-group-item+.list-group-item.active{margin-left:calc(-1*var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media(min-width: 992px){.list-group-horizontal-lg{flex-direction:row;-webkit-flex-direction:row}.list-group-horizontal-lg>.list-group-item.active{margin-top:0}.list-group-horizontal-lg>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-lg>.list-group-item+.list-group-item.active{margin-left:calc(-1*var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media(min-width: 1200px){.list-group-horizontal-xl{flex-direction:row;-webkit-flex-direction:row}.list-group-horizontal-xl>.list-group-item.active{margin-top:0}.list-group-horizontal-xl>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-xl>.list-group-item+.list-group-item.active{margin-left:calc(-1*var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}@media(min-width: 1400px){.list-group-horizontal-xxl{flex-direction:row;-webkit-flex-direction:row}.list-group-horizontal-xxl>.list-group-item.active{margin-top:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item{border-top-width:var(--bs-list-group-border-width);border-left-width:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item.active{margin-left:calc(-1*var(--bs-list-group-border-width));border-left-width:var(--bs-list-group-border-width)}}.list-group-flush>.list-group-item{border-width:0 0 var(--bs-list-group-border-width)}.list-group-flush>.list-group-item:last-child{border-bottom-width:0}.list-group-item-default{--bs-list-group-color: var(--bs-default-text-emphasis);--bs-list-group-bg: var(--bs-default-bg-subtle);--bs-list-group-border-color: var(--bs-default-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-default-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-default-border-subtle);--bs-list-group-active-color: var(--bs-default-bg-subtle);--bs-list-group-active-bg: var(--bs-default-text-emphasis);--bs-list-group-active-border-color: var(--bs-default-text-emphasis)}.list-group-item-primary{--bs-list-group-color: var(--bs-primary-text-emphasis);--bs-list-group-bg: var(--bs-primary-bg-subtle);--bs-list-group-border-color: var(--bs-primary-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-primary-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-primary-border-subtle);--bs-list-group-active-color: var(--bs-primary-bg-subtle);--bs-list-group-active-bg: var(--bs-primary-text-emphasis);--bs-list-group-active-border-color: var(--bs-primary-text-emphasis)}.list-group-item-secondary{--bs-list-group-color: var(--bs-secondary-text-emphasis);--bs-list-group-bg: var(--bs-secondary-bg-subtle);--bs-list-group-border-color: var(--bs-secondary-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-secondary-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-secondary-border-subtle);--bs-list-group-active-color: var(--bs-secondary-bg-subtle);--bs-list-group-active-bg: var(--bs-secondary-text-emphasis);--bs-list-group-active-border-color: var(--bs-secondary-text-emphasis)}.list-group-item-success{--bs-list-group-color: var(--bs-success-text-emphasis);--bs-list-group-bg: var(--bs-success-bg-subtle);--bs-list-group-border-color: var(--bs-success-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-success-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-success-border-subtle);--bs-list-group-active-color: var(--bs-success-bg-subtle);--bs-list-group-active-bg: var(--bs-success-text-emphasis);--bs-list-group-active-border-color: var(--bs-success-text-emphasis)}.list-group-item-info{--bs-list-group-color: var(--bs-info-text-emphasis);--bs-list-group-bg: var(--bs-info-bg-subtle);--bs-list-group-border-color: var(--bs-info-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-info-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-info-border-subtle);--bs-list-group-active-color: var(--bs-info-bg-subtle);--bs-list-group-active-bg: var(--bs-info-text-emphasis);--bs-list-group-active-border-color: var(--bs-info-text-emphasis)}.list-group-item-warning{--bs-list-group-color: var(--bs-warning-text-emphasis);--bs-list-group-bg: var(--bs-warning-bg-subtle);--bs-list-group-border-color: var(--bs-warning-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-warning-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-warning-border-subtle);--bs-list-group-active-color: var(--bs-warning-bg-subtle);--bs-list-group-active-bg: var(--bs-warning-text-emphasis);--bs-list-group-active-border-color: var(--bs-warning-text-emphasis)}.list-group-item-danger{--bs-list-group-color: var(--bs-danger-text-emphasis);--bs-list-group-bg: var(--bs-danger-bg-subtle);--bs-list-group-border-color: var(--bs-danger-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-danger-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-danger-border-subtle);--bs-list-group-active-color: var(--bs-danger-bg-subtle);--bs-list-group-active-bg: var(--bs-danger-text-emphasis);--bs-list-group-active-border-color: var(--bs-danger-text-emphasis)}.list-group-item-light{--bs-list-group-color: var(--bs-light-text-emphasis);--bs-list-group-bg: var(--bs-light-bg-subtle);--bs-list-group-border-color: var(--bs-light-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-light-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-light-border-subtle);--bs-list-group-active-color: var(--bs-light-bg-subtle);--bs-list-group-active-bg: var(--bs-light-text-emphasis);--bs-list-group-active-border-color: var(--bs-light-text-emphasis)}.list-group-item-dark{--bs-list-group-color: var(--bs-dark-text-emphasis);--bs-list-group-bg: var(--bs-dark-bg-subtle);--bs-list-group-border-color: var(--bs-dark-border-subtle);--bs-list-group-action-hover-color: var(--bs-emphasis-color);--bs-list-group-action-hover-bg: var(--bs-dark-border-subtle);--bs-list-group-action-active-color: var(--bs-emphasis-color);--bs-list-group-action-active-bg: var(--bs-dark-border-subtle);--bs-list-group-active-color: var(--bs-dark-bg-subtle);--bs-list-group-active-bg: var(--bs-dark-text-emphasis);--bs-list-group-active-border-color: var(--bs-dark-text-emphasis)}.btn-close{--bs-btn-close-color: #000;--bs-btn-close-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23000'%3e%3cpath d='M.293.293a1 1 0 0 1 1.414 0L8 6.586 14.293.293a1 1 0 1 1 1.414 1.414L9.414 8l6.293 6.293a1 1 0 0 1-1.414 1.414L8 9.414l-6.293 6.293a1 1 0 0 1-1.414-1.414L6.586 8 .293 1.707a1 1 0 0 1 0-1.414z'/%3e%3c/svg%3e");--bs-btn-close-opacity: 0.5;--bs-btn-close-hover-opacity: 0.75;--bs-btn-close-focus-shadow: 0 0 0 0.25rem rgba(39, 128, 227, 0.25);--bs-btn-close-focus-opacity: 1;--bs-btn-close-disabled-opacity: 0.25;--bs-btn-close-white-filter: invert(1) grayscale(100%) brightness(200%);box-sizing:content-box;width:1em;height:1em;padding:.25em .25em;color:var(--bs-btn-close-color);background:rgba(0,0,0,0) var(--bs-btn-close-bg) center/1em auto no-repeat;border:0;opacity:var(--bs-btn-close-opacity)}.btn-close:hover{color:var(--bs-btn-close-color);text-decoration:none;opacity:var(--bs-btn-close-hover-opacity)}.btn-close:focus{outline:0;box-shadow:var(--bs-btn-close-focus-shadow);opacity:var(--bs-btn-close-focus-opacity)}.btn-close:disabled,.btn-close.disabled{pointer-events:none;user-select:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;-o-user-select:none;opacity:var(--bs-btn-close-disabled-opacity)}.btn-close-white{filter:var(--bs-btn-close-white-filter)}[data-bs-theme=dark] .btn-close{filter:var(--bs-btn-close-white-filter)}.toast{--bs-toast-zindex: 1090;--bs-toast-padding-x: 0.75rem;--bs-toast-padding-y: 0.5rem;--bs-toast-spacing: 1.5rem;--bs-toast-max-width: 350px;--bs-toast-font-size:0.875rem;--bs-toast-color: ;--bs-toast-bg: rgba(255, 255, 255, 0.85);--bs-toast-border-width: 1px;--bs-toast-border-color: rgba(0, 0, 0, 0.175);--bs-toast-border-radius: 0.25rem;--bs-toast-box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15);--bs-toast-header-color: rgba(52, 58, 64, 0.75);--bs-toast-header-bg: rgba(255, 255, 255, 0.85);--bs-toast-header-border-color: rgba(0, 0, 0, 0.175);width:var(--bs-toast-max-width);max-width:100%;font-size:var(--bs-toast-font-size);color:var(--bs-toast-color);pointer-events:auto;background-color:var(--bs-toast-bg);background-clip:padding-box;border:var(--bs-toast-border-width) solid var(--bs-toast-border-color);box-shadow:var(--bs-toast-box-shadow)}.toast.showing{opacity:0}.toast:not(.show){display:none}.toast-container{--bs-toast-zindex: 1090;position:absolute;z-index:var(--bs-toast-zindex);width:max-content;width:-webkit-max-content;width:-moz-max-content;width:-ms-max-content;width:-o-max-content;max-width:100%;pointer-events:none}.toast-container>:not(:last-child){margin-bottom:var(--bs-toast-spacing)}.toast-header{display:flex;display:-webkit-flex;align-items:center;-webkit-align-items:center;padding:var(--bs-toast-padding-y) var(--bs-toast-padding-x);color:var(--bs-toast-header-color);background-color:var(--bs-toast-header-bg);background-clip:padding-box;border-bottom:var(--bs-toast-border-width) solid var(--bs-toast-header-border-color)}.toast-header .btn-close{margin-right:calc(-0.5*var(--bs-toast-padding-x));margin-left:var(--bs-toast-padding-x)}.toast-body{padding:var(--bs-toast-padding-x);word-wrap:break-word}.modal{--bs-modal-zindex: 1055;--bs-modal-width: 500px;--bs-modal-padding: 1rem;--bs-modal-margin: 0.5rem;--bs-modal-color: ;--bs-modal-bg: #fff;--bs-modal-border-color: rgba(0, 0, 0, 0.175);--bs-modal-border-width: 1px;--bs-modal-border-radius: 0.5rem;--bs-modal-box-shadow: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.075);--bs-modal-inner-border-radius: calc(0.5rem - 1px);--bs-modal-header-padding-x: 1rem;--bs-modal-header-padding-y: 1rem;--bs-modal-header-padding: 1rem 1rem;--bs-modal-header-border-color: #dee2e6;--bs-modal-header-border-width: 1px;--bs-modal-title-line-height: 1.5;--bs-modal-footer-gap: 0.5rem;--bs-modal-footer-bg: ;--bs-modal-footer-border-color: #dee2e6;--bs-modal-footer-border-width: 1px;position:fixed;top:0;left:0;z-index:var(--bs-modal-zindex);display:none;width:100%;height:100%;overflow-x:hidden;overflow-y:auto;outline:0}.modal-dialog{position:relative;width:auto;margin:var(--bs-modal-margin);pointer-events:none}.modal.fade .modal-dialog{transition:transform .3s ease-out;transform:translate(0, -50px)}@media(prefers-reduced-motion: reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{transform:none}.modal.modal-static .modal-dialog{transform:scale(1.02)}.modal-dialog-scrollable{height:calc(100% - var(--bs-modal-margin)*2)}.modal-dialog-scrollable .modal-content{max-height:100%;overflow:hidden}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:flex;display:-webkit-flex;align-items:center;-webkit-align-items:center;min-height:calc(100% - var(--bs-modal-margin)*2)}.modal-content{position:relative;display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;width:100%;color:var(--bs-modal-color);pointer-events:auto;background-color:var(--bs-modal-bg);background-clip:padding-box;border:var(--bs-modal-border-width) solid var(--bs-modal-border-color);outline:0}.modal-backdrop{--bs-backdrop-zindex: 1050;--bs-backdrop-bg: #000;--bs-backdrop-opacity: 0.5;position:fixed;top:0;left:0;z-index:var(--bs-backdrop-zindex);width:100vw;height:100vh;background-color:var(--bs-backdrop-bg)}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:var(--bs-backdrop-opacity)}.modal-header{display:flex;display:-webkit-flex;flex-shrink:0;-webkit-flex-shrink:0;align-items:center;-webkit-align-items:center;justify-content:space-between;-webkit-justify-content:space-between;padding:var(--bs-modal-header-padding);border-bottom:var(--bs-modal-header-border-width) solid var(--bs-modal-header-border-color)}.modal-header .btn-close{padding:calc(var(--bs-modal-header-padding-y)*.5) calc(var(--bs-modal-header-padding-x)*.5);margin:calc(-0.5*var(--bs-modal-header-padding-y)) calc(-0.5*var(--bs-modal-header-padding-x)) calc(-0.5*var(--bs-modal-header-padding-y)) auto}.modal-title{margin-bottom:0;line-height:var(--bs-modal-title-line-height)}.modal-body{position:relative;flex:1 1 auto;-webkit-flex:1 1 auto;padding:var(--bs-modal-padding)}.modal-footer{display:flex;display:-webkit-flex;flex-shrink:0;-webkit-flex-shrink:0;flex-wrap:wrap;-webkit-flex-wrap:wrap;align-items:center;-webkit-align-items:center;justify-content:flex-end;-webkit-justify-content:flex-end;padding:calc(var(--bs-modal-padding) - var(--bs-modal-footer-gap)*.5);background-color:var(--bs-modal-footer-bg);border-top:var(--bs-modal-footer-border-width) solid var(--bs-modal-footer-border-color)}.modal-footer>*{margin:calc(var(--bs-modal-footer-gap)*.5)}@media(min-width: 576px){.modal{--bs-modal-margin: 1.75rem;--bs-modal-box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15)}.modal-dialog{max-width:var(--bs-modal-width);margin-right:auto;margin-left:auto}.modal-sm{--bs-modal-width: 300px}}@media(min-width: 992px){.modal-lg,.modal-xl{--bs-modal-width: 800px}}@media(min-width: 1200px){.modal-xl{--bs-modal-width: 1140px}}.modal-fullscreen{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen .modal-content{height:100%;border:0}.modal-fullscreen .modal-body{overflow-y:auto}@media(max-width: 575.98px){.modal-fullscreen-sm-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-sm-down .modal-content{height:100%;border:0}.modal-fullscreen-sm-down .modal-body{overflow-y:auto}}@media(max-width: 767.98px){.modal-fullscreen-md-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-md-down .modal-content{height:100%;border:0}.modal-fullscreen-md-down .modal-body{overflow-y:auto}}@media(max-width: 991.98px){.modal-fullscreen-lg-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-lg-down .modal-content{height:100%;border:0}.modal-fullscreen-lg-down .modal-body{overflow-y:auto}}@media(max-width: 1199.98px){.modal-fullscreen-xl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xl-down .modal-content{height:100%;border:0}.modal-fullscreen-xl-down .modal-body{overflow-y:auto}}@media(max-width: 1399.98px){.modal-fullscreen-xxl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xxl-down .modal-content{height:100%;border:0}.modal-fullscreen-xxl-down .modal-body{overflow-y:auto}}.tooltip{--bs-tooltip-zindex: 1080;--bs-tooltip-max-width: 200px;--bs-tooltip-padding-x: 0.5rem;--bs-tooltip-padding-y: 0.25rem;--bs-tooltip-margin: ;--bs-tooltip-font-size:0.875rem;--bs-tooltip-color: #fff;--bs-tooltip-bg: #000;--bs-tooltip-border-radius: 0.25rem;--bs-tooltip-opacity: 0.9;--bs-tooltip-arrow-width: 0.8rem;--bs-tooltip-arrow-height: 0.4rem;z-index:var(--bs-tooltip-zindex);display:block;margin:var(--bs-tooltip-margin);font-family:"Source Sans Pro",-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;white-space:normal;word-spacing:normal;line-break:auto;font-size:var(--bs-tooltip-font-size);word-wrap:break-word;opacity:0}.tooltip.show{opacity:var(--bs-tooltip-opacity)}.tooltip .tooltip-arrow{display:block;width:var(--bs-tooltip-arrow-width);height:var(--bs-tooltip-arrow-height)}.tooltip .tooltip-arrow::before{position:absolute;content:"";border-color:rgba(0,0,0,0);border-style:solid}.bs-tooltip-top .tooltip-arrow,.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow{bottom:calc(-1*var(--bs-tooltip-arrow-height))}.bs-tooltip-top .tooltip-arrow::before,.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow::before{top:-1px;border-width:var(--bs-tooltip-arrow-height) calc(var(--bs-tooltip-arrow-width)*.5) 0;border-top-color:var(--bs-tooltip-bg)}.bs-tooltip-end .tooltip-arrow,.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow{left:calc(-1*var(--bs-tooltip-arrow-height));width:var(--bs-tooltip-arrow-height);height:var(--bs-tooltip-arrow-width)}.bs-tooltip-end .tooltip-arrow::before,.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow::before{right:-1px;border-width:calc(var(--bs-tooltip-arrow-width)*.5) var(--bs-tooltip-arrow-height) calc(var(--bs-tooltip-arrow-width)*.5) 0;border-right-color:var(--bs-tooltip-bg)}.bs-tooltip-bottom .tooltip-arrow,.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow{top:calc(-1*var(--bs-tooltip-arrow-height))}.bs-tooltip-bottom .tooltip-arrow::before,.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow::before{bottom:-1px;border-width:0 calc(var(--bs-tooltip-arrow-width)*.5) var(--bs-tooltip-arrow-height);border-bottom-color:var(--bs-tooltip-bg)}.bs-tooltip-start .tooltip-arrow,.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow{right:calc(-1*var(--bs-tooltip-arrow-height));width:var(--bs-tooltip-arrow-height);height:var(--bs-tooltip-arrow-width)}.bs-tooltip-start .tooltip-arrow::before,.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow::before{left:-1px;border-width:calc(var(--bs-tooltip-arrow-width)*.5) 0 calc(var(--bs-tooltip-arrow-width)*.5) var(--bs-tooltip-arrow-height);border-left-color:var(--bs-tooltip-bg)}.tooltip-inner{max-width:var(--bs-tooltip-max-width);padding:var(--bs-tooltip-padding-y) var(--bs-tooltip-padding-x);color:var(--bs-tooltip-color);text-align:center;background-color:var(--bs-tooltip-bg)}.popover{--bs-popover-zindex: 1070;--bs-popover-max-width: 276px;--bs-popover-font-size:0.875rem;--bs-popover-bg: #fff;--bs-popover-border-width: 1px;--bs-popover-border-color: rgba(0, 0, 0, 0.175);--bs-popover-border-radius: 0.5rem;--bs-popover-inner-border-radius: calc(0.5rem - 1px);--bs-popover-box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15);--bs-popover-header-padding-x: 1rem;--bs-popover-header-padding-y: 0.5rem;--bs-popover-header-font-size:1rem;--bs-popover-header-color: inherit;--bs-popover-header-bg: #e9ecef;--bs-popover-body-padding-x: 1rem;--bs-popover-body-padding-y: 1rem;--bs-popover-body-color: #343a40;--bs-popover-arrow-width: 1rem;--bs-popover-arrow-height: 0.5rem;--bs-popover-arrow-border: var(--bs-popover-border-color);z-index:var(--bs-popover-zindex);display:block;max-width:var(--bs-popover-max-width);font-family:"Source Sans Pro",-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;white-space:normal;word-spacing:normal;line-break:auto;font-size:var(--bs-popover-font-size);word-wrap:break-word;background-color:var(--bs-popover-bg);background-clip:padding-box;border:var(--bs-popover-border-width) solid var(--bs-popover-border-color)}.popover .popover-arrow{display:block;width:var(--bs-popover-arrow-width);height:var(--bs-popover-arrow-height)}.popover .popover-arrow::before,.popover .popover-arrow::after{position:absolute;display:block;content:"";border-color:rgba(0,0,0,0);border-style:solid;border-width:0}.bs-popover-top>.popover-arrow,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow{bottom:calc(-1*(var(--bs-popover-arrow-height)) - var(--bs-popover-border-width))}.bs-popover-top>.popover-arrow::before,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::before,.bs-popover-top>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::after{border-width:var(--bs-popover-arrow-height) calc(var(--bs-popover-arrow-width)*.5) 0}.bs-popover-top>.popover-arrow::before,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::before{bottom:0;border-top-color:var(--bs-popover-arrow-border)}.bs-popover-top>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::after{bottom:var(--bs-popover-border-width);border-top-color:var(--bs-popover-bg)}.bs-popover-end>.popover-arrow,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow{left:calc(-1*(var(--bs-popover-arrow-height)) - var(--bs-popover-border-width));width:var(--bs-popover-arrow-height);height:var(--bs-popover-arrow-width)}.bs-popover-end>.popover-arrow::before,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::before,.bs-popover-end>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::after{border-width:calc(var(--bs-popover-arrow-width)*.5) var(--bs-popover-arrow-height) calc(var(--bs-popover-arrow-width)*.5) 0}.bs-popover-end>.popover-arrow::before,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::before{left:0;border-right-color:var(--bs-popover-arrow-border)}.bs-popover-end>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::after{left:var(--bs-popover-border-width);border-right-color:var(--bs-popover-bg)}.bs-popover-bottom>.popover-arrow,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow{top:calc(-1*(var(--bs-popover-arrow-height)) - var(--bs-popover-border-width))}.bs-popover-bottom>.popover-arrow::before,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::before,.bs-popover-bottom>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::after{border-width:0 calc(var(--bs-popover-arrow-width)*.5) var(--bs-popover-arrow-height)}.bs-popover-bottom>.popover-arrow::before,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::before{top:0;border-bottom-color:var(--bs-popover-arrow-border)}.bs-popover-bottom>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::after{top:var(--bs-popover-border-width);border-bottom-color:var(--bs-popover-bg)}.bs-popover-bottom .popover-header::before,.bs-popover-auto[data-popper-placement^=bottom] .popover-header::before{position:absolute;top:0;left:50%;display:block;width:var(--bs-popover-arrow-width);margin-left:calc(-0.5*var(--bs-popover-arrow-width));content:"";border-bottom:var(--bs-popover-border-width) solid var(--bs-popover-header-bg)}.bs-popover-start>.popover-arrow,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow{right:calc(-1*(var(--bs-popover-arrow-height)) - var(--bs-popover-border-width));width:var(--bs-popover-arrow-height);height:var(--bs-popover-arrow-width)}.bs-popover-start>.popover-arrow::before,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::before,.bs-popover-start>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::after{border-width:calc(var(--bs-popover-arrow-width)*.5) 0 calc(var(--bs-popover-arrow-width)*.5) var(--bs-popover-arrow-height)}.bs-popover-start>.popover-arrow::before,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::before{right:0;border-left-color:var(--bs-popover-arrow-border)}.bs-popover-start>.popover-arrow::after,.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::after{right:var(--bs-popover-border-width);border-left-color:var(--bs-popover-bg)}.popover-header{padding:var(--bs-popover-header-padding-y) var(--bs-popover-header-padding-x);margin-bottom:0;font-size:var(--bs-popover-header-font-size);color:var(--bs-popover-header-color);background-color:var(--bs-popover-header-bg);border-bottom:var(--bs-popover-border-width) solid var(--bs-popover-border-color)}.popover-header:empty{display:none}.popover-body{padding:var(--bs-popover-body-padding-y) var(--bs-popover-body-padding-x);color:var(--bs-popover-body-color)}.carousel{position:relative}.carousel.pointer-event{touch-action:pan-y;-webkit-touch-action:pan-y;-moz-touch-action:pan-y;-ms-touch-action:pan-y;-o-touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;backface-visibility:hidden;-webkit-backface-visibility:hidden;-moz-backface-visibility:hidden;-ms-backface-visibility:hidden;-o-backface-visibility:hidden;transition:transform .6s ease-in-out}@media(prefers-reduced-motion: reduce){.carousel-item{transition:none}}.carousel-item.active,.carousel-item-next,.carousel-item-prev{display:block}.carousel-item-next:not(.carousel-item-start),.active.carousel-item-end{transform:translateX(100%)}.carousel-item-prev:not(.carousel-item-end),.active.carousel-item-start{transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;transform:none}.carousel-fade .carousel-item.active,.carousel-fade .carousel-item-next.carousel-item-start,.carousel-fade .carousel-item-prev.carousel-item-end{z-index:1;opacity:1}.carousel-fade .active.carousel-item-start,.carousel-fade .active.carousel-item-end{z-index:0;opacity:0;transition:opacity 0s .6s}@media(prefers-reduced-motion: reduce){.carousel-fade .active.carousel-item-start,.carousel-fade .active.carousel-item-end{transition:none}}.carousel-control-prev,.carousel-control-next{position:absolute;top:0;bottom:0;z-index:1;display:flex;display:-webkit-flex;align-items:center;-webkit-align-items:center;justify-content:center;-webkit-justify-content:center;width:15%;padding:0;color:#fff;text-align:center;background:none;border:0;opacity:.5;transition:opacity .15s ease}@media(prefers-reduced-motion: reduce){.carousel-control-prev,.carousel-control-next{transition:none}}.carousel-control-prev:hover,.carousel-control-prev:focus,.carousel-control-next:hover,.carousel-control-next:focus{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-prev-icon,.carousel-control-next-icon{display:inline-block;width:2rem;height:2rem;background-repeat:no-repeat;background-position:50%;background-size:100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M11.354 1.646a.5.5 0 0 1 0 .708L5.707 8l5.647 5.646a.5.5 0 0 1-.708.708l-6-6a.5.5 0 0 1 0-.708l6-6a.5.5 0 0 1 .708 0z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M4.646 1.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1 0 .708l-6 6a.5.5 0 0 1-.708-.708L10.293 8 4.646 2.354a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:2;display:flex;display:-webkit-flex;justify-content:center;-webkit-justify-content:center;padding:0;margin-right:15%;margin-bottom:1rem;margin-left:15%}.carousel-indicators [data-bs-target]{box-sizing:content-box;flex:0 1 auto;-webkit-flex:0 1 auto;width:30px;height:3px;padding:0;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border:0;border-top:10px solid rgba(0,0,0,0);border-bottom:10px solid rgba(0,0,0,0);opacity:.5;transition:opacity .6s ease}@media(prefers-reduced-motion: reduce){.carousel-indicators [data-bs-target]{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:1.25rem;left:15%;padding-top:1.25rem;padding-bottom:1.25rem;color:#fff;text-align:center}.carousel-dark .carousel-control-prev-icon,.carousel-dark .carousel-control-next-icon{filter:invert(1) grayscale(100)}.carousel-dark .carousel-indicators [data-bs-target]{background-color:#000}.carousel-dark .carousel-caption{color:#000}[data-bs-theme=dark] .carousel .carousel-control-prev-icon,[data-bs-theme=dark] .carousel .carousel-control-next-icon,[data-bs-theme=dark].carousel .carousel-control-prev-icon,[data-bs-theme=dark].carousel .carousel-control-next-icon{filter:invert(1) grayscale(100)}[data-bs-theme=dark] .carousel .carousel-indicators [data-bs-target],[data-bs-theme=dark].carousel .carousel-indicators [data-bs-target]{background-color:#000}[data-bs-theme=dark] .carousel .carousel-caption,[data-bs-theme=dark].carousel .carousel-caption{color:#000}.spinner-grow,.spinner-border{display:inline-block;width:var(--bs-spinner-width);height:var(--bs-spinner-height);vertical-align:var(--bs-spinner-vertical-align);border-radius:50%;animation:var(--bs-spinner-animation-speed) linear infinite var(--bs-spinner-animation-name)}@keyframes spinner-border{to{transform:rotate(360deg) /* rtl:ignore */}}.spinner-border{--bs-spinner-width: 2rem;--bs-spinner-height: 2rem;--bs-spinner-vertical-align: -0.125em;--bs-spinner-border-width: 0.25em;--bs-spinner-animation-speed: 0.75s;--bs-spinner-animation-name: spinner-border;border:var(--bs-spinner-border-width) solid currentcolor;border-right-color:rgba(0,0,0,0)}.spinner-border-sm{--bs-spinner-width: 1rem;--bs-spinner-height: 1rem;--bs-spinner-border-width: 0.2em}@keyframes spinner-grow{0%{transform:scale(0)}50%{opacity:1;transform:none}}.spinner-grow{--bs-spinner-width: 2rem;--bs-spinner-height: 2rem;--bs-spinner-vertical-align: -0.125em;--bs-spinner-animation-speed: 0.75s;--bs-spinner-animation-name: spinner-grow;background-color:currentcolor;opacity:0}.spinner-grow-sm{--bs-spinner-width: 1rem;--bs-spinner-height: 1rem}@media(prefers-reduced-motion: reduce){.spinner-border,.spinner-grow{--bs-spinner-animation-speed: 1.5s}}.offcanvas,.offcanvas-xxl,.offcanvas-xl,.offcanvas-lg,.offcanvas-md,.offcanvas-sm{--bs-offcanvas-zindex: 1045;--bs-offcanvas-width: 400px;--bs-offcanvas-height: 30vh;--bs-offcanvas-padding-x: 1rem;--bs-offcanvas-padding-y: 1rem;--bs-offcanvas-color: #343a40;--bs-offcanvas-bg: #fff;--bs-offcanvas-border-width: 1px;--bs-offcanvas-border-color: rgba(0, 0, 0, 0.175);--bs-offcanvas-box-shadow: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.075);--bs-offcanvas-transition: transform 0.3s ease-in-out;--bs-offcanvas-title-line-height: 1.5}@media(max-width: 575.98px){.offcanvas-sm{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media(max-width: 575.98px)and (prefers-reduced-motion: reduce){.offcanvas-sm{transition:none}}@media(max-width: 575.98px){.offcanvas-sm.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-sm.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-sm.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-sm.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-sm.showing,.offcanvas-sm.show:not(.hiding){transform:none}.offcanvas-sm.showing,.offcanvas-sm.hiding,.offcanvas-sm.show{visibility:visible}}@media(min-width: 576px){.offcanvas-sm{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:rgba(0,0,0,0) !important}.offcanvas-sm .offcanvas-header{display:none}.offcanvas-sm .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible;background-color:rgba(0,0,0,0) !important}}@media(max-width: 767.98px){.offcanvas-md{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media(max-width: 767.98px)and (prefers-reduced-motion: reduce){.offcanvas-md{transition:none}}@media(max-width: 767.98px){.offcanvas-md.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-md.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-md.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-md.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-md.showing,.offcanvas-md.show:not(.hiding){transform:none}.offcanvas-md.showing,.offcanvas-md.hiding,.offcanvas-md.show{visibility:visible}}@media(min-width: 768px){.offcanvas-md{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:rgba(0,0,0,0) !important}.offcanvas-md .offcanvas-header{display:none}.offcanvas-md .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible;background-color:rgba(0,0,0,0) !important}}@media(max-width: 991.98px){.offcanvas-lg{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media(max-width: 991.98px)and (prefers-reduced-motion: reduce){.offcanvas-lg{transition:none}}@media(max-width: 991.98px){.offcanvas-lg.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-lg.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-lg.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-lg.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-lg.showing,.offcanvas-lg.show:not(.hiding){transform:none}.offcanvas-lg.showing,.offcanvas-lg.hiding,.offcanvas-lg.show{visibility:visible}}@media(min-width: 992px){.offcanvas-lg{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:rgba(0,0,0,0) !important}.offcanvas-lg .offcanvas-header{display:none}.offcanvas-lg .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible;background-color:rgba(0,0,0,0) !important}}@media(max-width: 1199.98px){.offcanvas-xl{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media(max-width: 1199.98px)and (prefers-reduced-motion: reduce){.offcanvas-xl{transition:none}}@media(max-width: 1199.98px){.offcanvas-xl.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-xl.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-xl.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-xl.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-xl.showing,.offcanvas-xl.show:not(.hiding){transform:none}.offcanvas-xl.showing,.offcanvas-xl.hiding,.offcanvas-xl.show{visibility:visible}}@media(min-width: 1200px){.offcanvas-xl{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:rgba(0,0,0,0) !important}.offcanvas-xl .offcanvas-header{display:none}.offcanvas-xl .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible;background-color:rgba(0,0,0,0) !important}}@media(max-width: 1399.98px){.offcanvas-xxl{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}}@media(max-width: 1399.98px)and (prefers-reduced-motion: reduce){.offcanvas-xxl{transition:none}}@media(max-width: 1399.98px){.offcanvas-xxl.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas-xxl.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas-xxl.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas-xxl.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas-xxl.showing,.offcanvas-xxl.show:not(.hiding){transform:none}.offcanvas-xxl.showing,.offcanvas-xxl.hiding,.offcanvas-xxl.show{visibility:visible}}@media(min-width: 1400px){.offcanvas-xxl{--bs-offcanvas-height: auto;--bs-offcanvas-border-width: 0;background-color:rgba(0,0,0,0) !important}.offcanvas-xxl .offcanvas-header{display:none}.offcanvas-xxl .offcanvas-body{display:flex;display:-webkit-flex;flex-grow:0;-webkit-flex-grow:0;padding:0;overflow-y:visible;background-color:rgba(0,0,0,0) !important}}.offcanvas{position:fixed;bottom:0;z-index:var(--bs-offcanvas-zindex);display:flex;display:-webkit-flex;flex-direction:column;-webkit-flex-direction:column;max-width:100%;color:var(--bs-offcanvas-color);visibility:hidden;background-color:var(--bs-offcanvas-bg);background-clip:padding-box;outline:0;transition:var(--bs-offcanvas-transition)}@media(prefers-reduced-motion: reduce){.offcanvas{transition:none}}.offcanvas.offcanvas-start{top:0;left:0;width:var(--bs-offcanvas-width);border-right:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(-100%)}.offcanvas.offcanvas-end{top:0;right:0;width:var(--bs-offcanvas-width);border-left:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateX(100%)}.offcanvas.offcanvas-top{top:0;right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-bottom:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(-100%)}.offcanvas.offcanvas-bottom{right:0;left:0;height:var(--bs-offcanvas-height);max-height:100%;border-top:var(--bs-offcanvas-border-width) solid var(--bs-offcanvas-border-color);transform:translateY(100%)}.offcanvas.showing,.offcanvas.show:not(.hiding){transform:none}.offcanvas.showing,.offcanvas.hiding,.offcanvas.show{visibility:visible}.offcanvas-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.offcanvas-backdrop.fade{opacity:0}.offcanvas-backdrop.show{opacity:.5}.offcanvas-header{display:flex;display:-webkit-flex;align-items:center;-webkit-align-items:center;justify-content:space-between;-webkit-justify-content:space-between;padding:var(--bs-offcanvas-padding-y) var(--bs-offcanvas-padding-x)}.offcanvas-header .btn-close{padding:calc(var(--bs-offcanvas-padding-y)*.5) calc(var(--bs-offcanvas-padding-x)*.5);margin-top:calc(-0.5*var(--bs-offcanvas-padding-y));margin-right:calc(-0.5*var(--bs-offcanvas-padding-x));margin-bottom:calc(-0.5*var(--bs-offcanvas-padding-y))}.offcanvas-title{margin-bottom:0;line-height:var(--bs-offcanvas-title-line-height)}.offcanvas-body{flex-grow:1;-webkit-flex-grow:1;padding:var(--bs-offcanvas-padding-y) var(--bs-offcanvas-padding-x);overflow-y:auto}.placeholder{display:inline-block;min-height:1em;vertical-align:middle;cursor:wait;background-color:currentcolor;opacity:.5}.placeholder.btn::before{display:inline-block;content:""}.placeholder-xs{min-height:.6em}.placeholder-sm{min-height:.8em}.placeholder-lg{min-height:1.2em}.placeholder-glow .placeholder{animation:placeholder-glow 2s ease-in-out infinite}@keyframes placeholder-glow{50%{opacity:.2}}.placeholder-wave{mask-image:linear-gradient(130deg, #000 55%, rgba(0, 0, 0, 0.8) 75%, #000 95%);-webkit-mask-image:linear-gradient(130deg, #000 55%, rgba(0, 0, 0, 0.8) 75%, #000 95%);mask-size:200% 100%;-webkit-mask-size:200% 100%;animation:placeholder-wave 2s linear infinite}@keyframes placeholder-wave{100%{mask-position:-200% 0%;-webkit-mask-position:-200% 0%}}.clearfix::after{display:block;clear:both;content:""}.text-bg-default{color:#fff !important;background-color:RGBA(var(--bs-default-rgb), var(--bs-bg-opacity, 1)) !important}.text-bg-primary{color:#fff !important;background-color:RGBA(var(--bs-primary-rgb), var(--bs-bg-opacity, 1)) !important}.text-bg-secondary{color:#fff !important;background-color:RGBA(var(--bs-secondary-rgb), var(--bs-bg-opacity, 1)) !important}.text-bg-success{color:#fff !important;background-color:RGBA(var(--bs-success-rgb), var(--bs-bg-opacity, 1)) !important}.text-bg-info{color:#fff !important;background-color:RGBA(var(--bs-info-rgb), var(--bs-bg-opacity, 1)) !important}.text-bg-warning{color:#fff !important;background-color:RGBA(var(--bs-warning-rgb), var(--bs-bg-opacity, 1)) !important}.text-bg-danger{color:#fff !important;background-color:RGBA(var(--bs-danger-rgb), var(--bs-bg-opacity, 1)) !important}.text-bg-light{color:#000 !important;background-color:RGBA(var(--bs-light-rgb), var(--bs-bg-opacity, 1)) !important}.text-bg-dark{color:#fff !important;background-color:RGBA(var(--bs-dark-rgb), var(--bs-bg-opacity, 1)) !important}.link-default{color:RGBA(var(--bs-default-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-default-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-default:hover,.link-default:focus{color:RGBA(42, 46, 51, var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(42, 46, 51, var(--bs-link-underline-opacity, 1)) !important}.link-primary{color:RGBA(var(--bs-primary-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-primary-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-primary:hover,.link-primary:focus{color:RGBA(31, 102, 182, var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(31, 102, 182, var(--bs-link-underline-opacity, 1)) !important}.link-secondary{color:RGBA(var(--bs-secondary-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-secondary-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-secondary:hover,.link-secondary:focus{color:RGBA(42, 46, 51, var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(42, 46, 51, var(--bs-link-underline-opacity, 1)) !important}.link-success{color:RGBA(var(--bs-success-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-success-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-success:hover,.link-success:focus{color:RGBA(50, 146, 19, var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(50, 146, 19, var(--bs-link-underline-opacity, 1)) !important}.link-info{color:RGBA(var(--bs-info-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-info-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-info:hover,.link-info:focus{color:RGBA(122, 67, 150, var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(122, 67, 150, var(--bs-link-underline-opacity, 1)) !important}.link-warning{color:RGBA(var(--bs-warning-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-warning-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-warning:hover,.link-warning:focus{color:RGBA(204, 94, 19, var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(204, 94, 19, var(--bs-link-underline-opacity, 1)) !important}.link-danger{color:RGBA(var(--bs-danger-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-danger-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-danger:hover,.link-danger:focus{color:RGBA(204, 0, 46, var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(204, 0, 46, var(--bs-link-underline-opacity, 1)) !important}.link-light{color:RGBA(var(--bs-light-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-light-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-light:hover,.link-light:focus{color:RGBA(249, 250, 251, var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(249, 250, 251, var(--bs-link-underline-opacity, 1)) !important}.link-dark{color:RGBA(var(--bs-dark-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-dark-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-dark:hover,.link-dark:focus{color:RGBA(42, 46, 51, var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(42, 46, 51, var(--bs-link-underline-opacity, 1)) !important}.link-body-emphasis{color:RGBA(var(--bs-emphasis-color-rgb), var(--bs-link-opacity, 1)) !important;text-decoration-color:RGBA(var(--bs-emphasis-color-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-body-emphasis:hover,.link-body-emphasis:focus{color:RGBA(var(--bs-emphasis-color-rgb), var(--bs-link-opacity, 0.75)) !important;text-decoration-color:RGBA(var(--bs-emphasis-color-rgb), var(--bs-link-underline-opacity, 0.75)) !important}.focus-ring:focus{outline:0;box-shadow:var(--bs-focus-ring-x, 0) var(--bs-focus-ring-y, 0) var(--bs-focus-ring-blur, 0) var(--bs-focus-ring-width) var(--bs-focus-ring-color)}.icon-link{display:inline-flex;gap:.375rem;align-items:center;-webkit-align-items:center;text-decoration-color:rgba(var(--bs-link-color-rgb), var(--bs-link-opacity, 0.5));text-underline-offset:.25em;backface-visibility:hidden;-webkit-backface-visibility:hidden;-moz-backface-visibility:hidden;-ms-backface-visibility:hidden;-o-backface-visibility:hidden}.icon-link>.bi{flex-shrink:0;-webkit-flex-shrink:0;width:1em;height:1em;fill:currentcolor;transition:.2s ease-in-out transform}@media(prefers-reduced-motion: reduce){.icon-link>.bi{transition:none}}.icon-link-hover:hover>.bi,.icon-link-hover:focus-visible>.bi{transform:var(--bs-icon-link-transform, translate3d(0.25em, 0, 0))}.ratio{position:relative;width:100%}.ratio::before{display:block;padding-top:var(--bs-aspect-ratio);content:""}.ratio>*{position:absolute;top:0;left:0;width:100%;height:100%}.ratio-1x1{--bs-aspect-ratio: 100%}.ratio-4x3{--bs-aspect-ratio: 75%}.ratio-16x9{--bs-aspect-ratio: 56.25%}.ratio-21x9{--bs-aspect-ratio: 42.8571428571%}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}.sticky-top{position:sticky;top:0;z-index:1020}.sticky-bottom{position:sticky;bottom:0;z-index:1020}@media(min-width: 576px){.sticky-sm-top{position:sticky;top:0;z-index:1020}.sticky-sm-bottom{position:sticky;bottom:0;z-index:1020}}@media(min-width: 768px){.sticky-md-top{position:sticky;top:0;z-index:1020}.sticky-md-bottom{position:sticky;bottom:0;z-index:1020}}@media(min-width: 992px){.sticky-lg-top{position:sticky;top:0;z-index:1020}.sticky-lg-bottom{position:sticky;bottom:0;z-index:1020}}@media(min-width: 1200px){.sticky-xl-top{position:sticky;top:0;z-index:1020}.sticky-xl-bottom{position:sticky;bottom:0;z-index:1020}}@media(min-width: 1400px){.sticky-xxl-top{position:sticky;top:0;z-index:1020}.sticky-xxl-bottom{position:sticky;bottom:0;z-index:1020}}.hstack{display:flex;display:-webkit-flex;flex-direction:row;-webkit-flex-direction:row;align-items:center;-webkit-align-items:center;align-self:stretch;-webkit-align-self:stretch}.vstack{display:flex;display:-webkit-flex;flex:1 1 auto;-webkit-flex:1 1 auto;flex-direction:column;-webkit-flex-direction:column;align-self:stretch;-webkit-align-self:stretch}.visually-hidden,.visually-hidden-focusable:not(:focus):not(:focus-within){width:1px !important;height:1px !important;padding:0 !important;margin:-1px !important;overflow:hidden !important;clip:rect(0, 0, 0, 0) !important;white-space:nowrap !important;border:0 !important}.visually-hidden:not(caption),.visually-hidden-focusable:not(:focus):not(:focus-within):not(caption){position:absolute !important}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;content:""}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.vr{display:inline-block;align-self:stretch;-webkit-align-self:stretch;width:1px;min-height:1em;background-color:currentcolor;opacity:.25}.align-baseline{vertical-align:baseline !important}.align-top{vertical-align:top !important}.align-middle{vertical-align:middle !important}.align-bottom{vertical-align:bottom !important}.align-text-bottom{vertical-align:text-bottom !important}.align-text-top{vertical-align:text-top !important}.float-start{float:left !important}.float-end{float:right !important}.float-none{float:none !important}.object-fit-contain{object-fit:contain !important}.object-fit-cover{object-fit:cover !important}.object-fit-fill{object-fit:fill !important}.object-fit-scale{object-fit:scale-down !important}.object-fit-none{object-fit:none !important}.opacity-0{opacity:0 !important}.opacity-25{opacity:.25 !important}.opacity-50{opacity:.5 !important}.opacity-75{opacity:.75 !important}.opacity-100{opacity:1 !important}.overflow-auto{overflow:auto !important}.overflow-hidden{overflow:hidden !important}.overflow-visible{overflow:visible !important}.overflow-scroll{overflow:scroll !important}.overflow-x-auto{overflow-x:auto !important}.overflow-x-hidden{overflow-x:hidden !important}.overflow-x-visible{overflow-x:visible !important}.overflow-x-scroll{overflow-x:scroll !important}.overflow-y-auto{overflow-y:auto !important}.overflow-y-hidden{overflow-y:hidden !important}.overflow-y-visible{overflow-y:visible !important}.overflow-y-scroll{overflow-y:scroll !important}.d-inline{display:inline !important}.d-inline-block{display:inline-block !important}.d-block{display:block !important}.d-grid{display:grid !important}.d-inline-grid{display:inline-grid !important}.d-table{display:table !important}.d-table-row{display:table-row !important}.d-table-cell{display:table-cell !important}.d-flex{display:flex !important}.d-inline-flex{display:inline-flex !important}.d-none{display:none !important}.shadow{box-shadow:0 .5rem 1rem rgba(0,0,0,.15) !important}.shadow-sm{box-shadow:0 .125rem .25rem rgba(0,0,0,.075) !important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,.175) !important}.shadow-none{box-shadow:none !important}.focus-ring-default{--bs-focus-ring-color: rgba(var(--bs-default-rgb), var(--bs-focus-ring-opacity))}.focus-ring-primary{--bs-focus-ring-color: rgba(var(--bs-primary-rgb), var(--bs-focus-ring-opacity))}.focus-ring-secondary{--bs-focus-ring-color: rgba(var(--bs-secondary-rgb), var(--bs-focus-ring-opacity))}.focus-ring-success{--bs-focus-ring-color: rgba(var(--bs-success-rgb), var(--bs-focus-ring-opacity))}.focus-ring-info{--bs-focus-ring-color: rgba(var(--bs-info-rgb), var(--bs-focus-ring-opacity))}.focus-ring-warning{--bs-focus-ring-color: rgba(var(--bs-warning-rgb), var(--bs-focus-ring-opacity))}.focus-ring-danger{--bs-focus-ring-color: rgba(var(--bs-danger-rgb), var(--bs-focus-ring-opacity))}.focus-ring-light{--bs-focus-ring-color: rgba(var(--bs-light-rgb), var(--bs-focus-ring-opacity))}.focus-ring-dark{--bs-focus-ring-color: rgba(var(--bs-dark-rgb), var(--bs-focus-ring-opacity))}.position-static{position:static !important}.position-relative{position:relative !important}.position-absolute{position:absolute !important}.position-fixed{position:fixed !important}.position-sticky{position:sticky !important}.top-0{top:0 !important}.top-50{top:50% !important}.top-100{top:100% !important}.bottom-0{bottom:0 !important}.bottom-50{bottom:50% !important}.bottom-100{bottom:100% !important}.start-0{left:0 !important}.start-50{left:50% !important}.start-100{left:100% !important}.end-0{right:0 !important}.end-50{right:50% !important}.end-100{right:100% !important}.translate-middle{transform:translate(-50%, -50%) !important}.translate-middle-x{transform:translateX(-50%) !important}.translate-middle-y{transform:translateY(-50%) !important}.border{border:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color) !important}.border-0{border:0 !important}.border-top{border-top:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color) !important}.border-top-0{border-top:0 !important}.border-end{border-right:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color) !important}.border-end-0{border-right:0 !important}.border-bottom{border-bottom:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color) !important}.border-bottom-0{border-bottom:0 !important}.border-start{border-left:var(--bs-border-width) var(--bs-border-style) var(--bs-border-color) !important}.border-start-0{border-left:0 !important}.border-default{--bs-border-opacity: 1;border-color:rgba(var(--bs-default-rgb), var(--bs-border-opacity)) !important}.border-primary{--bs-border-opacity: 1;border-color:rgba(var(--bs-primary-rgb), var(--bs-border-opacity)) !important}.border-secondary{--bs-border-opacity: 1;border-color:rgba(var(--bs-secondary-rgb), var(--bs-border-opacity)) !important}.border-success{--bs-border-opacity: 1;border-color:rgba(var(--bs-success-rgb), var(--bs-border-opacity)) !important}.border-info{--bs-border-opacity: 1;border-color:rgba(var(--bs-info-rgb), var(--bs-border-opacity)) !important}.border-warning{--bs-border-opacity: 1;border-color:rgba(var(--bs-warning-rgb), var(--bs-border-opacity)) !important}.border-danger{--bs-border-opacity: 1;border-color:rgba(var(--bs-danger-rgb), var(--bs-border-opacity)) !important}.border-light{--bs-border-opacity: 1;border-color:rgba(var(--bs-light-rgb), var(--bs-border-opacity)) !important}.border-dark{--bs-border-opacity: 1;border-color:rgba(var(--bs-dark-rgb), var(--bs-border-opacity)) !important}.border-black{--bs-border-opacity: 1;border-color:rgba(var(--bs-black-rgb), var(--bs-border-opacity)) !important}.border-white{--bs-border-opacity: 1;border-color:rgba(var(--bs-white-rgb), var(--bs-border-opacity)) !important}.border-primary-subtle{border-color:var(--bs-primary-border-subtle) !important}.border-secondary-subtle{border-color:var(--bs-secondary-border-subtle) !important}.border-success-subtle{border-color:var(--bs-success-border-subtle) !important}.border-info-subtle{border-color:var(--bs-info-border-subtle) !important}.border-warning-subtle{border-color:var(--bs-warning-border-subtle) !important}.border-danger-subtle{border-color:var(--bs-danger-border-subtle) !important}.border-light-subtle{border-color:var(--bs-light-border-subtle) !important}.border-dark-subtle{border-color:var(--bs-dark-border-subtle) !important}.border-1{border-width:1px !important}.border-2{border-width:2px !important}.border-3{border-width:3px !important}.border-4{border-width:4px !important}.border-5{border-width:5px !important}.border-opacity-10{--bs-border-opacity: 0.1}.border-opacity-25{--bs-border-opacity: 0.25}.border-opacity-50{--bs-border-opacity: 0.5}.border-opacity-75{--bs-border-opacity: 0.75}.border-opacity-100{--bs-border-opacity: 1}.w-25{width:25% !important}.w-50{width:50% !important}.w-75{width:75% !important}.w-100{width:100% !important}.w-auto{width:auto !important}.mw-100{max-width:100% !important}.vw-100{width:100vw !important}.min-vw-100{min-width:100vw !important}.h-25{height:25% !important}.h-50{height:50% !important}.h-75{height:75% !important}.h-100{height:100% !important}.h-auto{height:auto !important}.mh-100{max-height:100% !important}.vh-100{height:100vh !important}.min-vh-100{min-height:100vh !important}.flex-fill{flex:1 1 auto !important}.flex-row{flex-direction:row !important}.flex-column{flex-direction:column !important}.flex-row-reverse{flex-direction:row-reverse !important}.flex-column-reverse{flex-direction:column-reverse !important}.flex-grow-0{flex-grow:0 !important}.flex-grow-1{flex-grow:1 !important}.flex-shrink-0{flex-shrink:0 !important}.flex-shrink-1{flex-shrink:1 !important}.flex-wrap{flex-wrap:wrap !important}.flex-nowrap{flex-wrap:nowrap !important}.flex-wrap-reverse{flex-wrap:wrap-reverse !important}.justify-content-start{justify-content:flex-start !important}.justify-content-end{justify-content:flex-end !important}.justify-content-center{justify-content:center !important}.justify-content-between{justify-content:space-between !important}.justify-content-around{justify-content:space-around !important}.justify-content-evenly{justify-content:space-evenly !important}.align-items-start{align-items:flex-start !important}.align-items-end{align-items:flex-end !important}.align-items-center{align-items:center !important}.align-items-baseline{align-items:baseline !important}.align-items-stretch{align-items:stretch !important}.align-content-start{align-content:flex-start !important}.align-content-end{align-content:flex-end !important}.align-content-center{align-content:center !important}.align-content-between{align-content:space-between !important}.align-content-around{align-content:space-around !important}.align-content-stretch{align-content:stretch !important}.align-self-auto{align-self:auto !important}.align-self-start{align-self:flex-start !important}.align-self-end{align-self:flex-end !important}.align-self-center{align-self:center !important}.align-self-baseline{align-self:baseline !important}.align-self-stretch{align-self:stretch !important}.order-first{order:-1 !important}.order-0{order:0 !important}.order-1{order:1 !important}.order-2{order:2 !important}.order-3{order:3 !important}.order-4{order:4 !important}.order-5{order:5 !important}.order-last{order:6 !important}.m-0{margin:0 !important}.m-1{margin:.25rem !important}.m-2{margin:.5rem !important}.m-3{margin:1rem !important}.m-4{margin:1.5rem !important}.m-5{margin:3rem !important}.m-auto{margin:auto !important}.mx-0{margin-right:0 !important;margin-left:0 !important}.mx-1{margin-right:.25rem !important;margin-left:.25rem !important}.mx-2{margin-right:.5rem !important;margin-left:.5rem !important}.mx-3{margin-right:1rem !important;margin-left:1rem !important}.mx-4{margin-right:1.5rem !important;margin-left:1.5rem !important}.mx-5{margin-right:3rem !important;margin-left:3rem !important}.mx-auto{margin-right:auto !important;margin-left:auto !important}.my-0{margin-top:0 !important;margin-bottom:0 !important}.my-1{margin-top:.25rem !important;margin-bottom:.25rem !important}.my-2{margin-top:.5rem !important;margin-bottom:.5rem !important}.my-3{margin-top:1rem !important;margin-bottom:1rem !important}.my-4{margin-top:1.5rem !important;margin-bottom:1.5rem !important}.my-5{margin-top:3rem !important;margin-bottom:3rem !important}.my-auto{margin-top:auto !important;margin-bottom:auto !important}.mt-0{margin-top:0 !important}.mt-1{margin-top:.25rem !important}.mt-2{margin-top:.5rem !important}.mt-3{margin-top:1rem !important}.mt-4{margin-top:1.5rem !important}.mt-5{margin-top:3rem !important}.mt-auto{margin-top:auto !important}.me-0{margin-right:0 !important}.me-1{margin-right:.25rem !important}.me-2{margin-right:.5rem !important}.me-3{margin-right:1rem !important}.me-4{margin-right:1.5rem !important}.me-5{margin-right:3rem !important}.me-auto{margin-right:auto !important}.mb-0{margin-bottom:0 !important}.mb-1{margin-bottom:.25rem !important}.mb-2{margin-bottom:.5rem !important}.mb-3{margin-bottom:1rem !important}.mb-4{margin-bottom:1.5rem !important}.mb-5{margin-bottom:3rem !important}.mb-auto{margin-bottom:auto !important}.ms-0{margin-left:0 !important}.ms-1{margin-left:.25rem !important}.ms-2{margin-left:.5rem !important}.ms-3{margin-left:1rem !important}.ms-4{margin-left:1.5rem !important}.ms-5{margin-left:3rem !important}.ms-auto{margin-left:auto !important}.p-0{padding:0 !important}.p-1{padding:.25rem !important}.p-2{padding:.5rem !important}.p-3{padding:1rem !important}.p-4{padding:1.5rem !important}.p-5{padding:3rem !important}.px-0{padding-right:0 !important;padding-left:0 !important}.px-1{padding-right:.25rem !important;padding-left:.25rem !important}.px-2{padding-right:.5rem !important;padding-left:.5rem !important}.px-3{padding-right:1rem !important;padding-left:1rem !important}.px-4{padding-right:1.5rem !important;padding-left:1.5rem !important}.px-5{padding-right:3rem !important;padding-left:3rem !important}.py-0{padding-top:0 !important;padding-bottom:0 !important}.py-1{padding-top:.25rem !important;padding-bottom:.25rem !important}.py-2{padding-top:.5rem !important;padding-bottom:.5rem !important}.py-3{padding-top:1rem !important;padding-bottom:1rem !important}.py-4{padding-top:1.5rem !important;padding-bottom:1.5rem !important}.py-5{padding-top:3rem !important;padding-bottom:3rem !important}.pt-0{padding-top:0 !important}.pt-1{padding-top:.25rem !important}.pt-2{padding-top:.5rem !important}.pt-3{padding-top:1rem !important}.pt-4{padding-top:1.5rem !important}.pt-5{padding-top:3rem !important}.pe-0{padding-right:0 !important}.pe-1{padding-right:.25rem !important}.pe-2{padding-right:.5rem !important}.pe-3{padding-right:1rem !important}.pe-4{padding-right:1.5rem !important}.pe-5{padding-right:3rem !important}.pb-0{padding-bottom:0 !important}.pb-1{padding-bottom:.25rem !important}.pb-2{padding-bottom:.5rem !important}.pb-3{padding-bottom:1rem !important}.pb-4{padding-bottom:1.5rem !important}.pb-5{padding-bottom:3rem !important}.ps-0{padding-left:0 !important}.ps-1{padding-left:.25rem !important}.ps-2{padding-left:.5rem !important}.ps-3{padding-left:1rem !important}.ps-4{padding-left:1.5rem !important}.ps-5{padding-left:3rem !important}.gap-0{gap:0 !important}.gap-1{gap:.25rem !important}.gap-2{gap:.5rem !important}.gap-3{gap:1rem !important}.gap-4{gap:1.5rem !important}.gap-5{gap:3rem !important}.row-gap-0{row-gap:0 !important}.row-gap-1{row-gap:.25rem !important}.row-gap-2{row-gap:.5rem !important}.row-gap-3{row-gap:1rem !important}.row-gap-4{row-gap:1.5rem !important}.row-gap-5{row-gap:3rem !important}.column-gap-0{column-gap:0 !important}.column-gap-1{column-gap:.25rem !important}.column-gap-2{column-gap:.5rem !important}.column-gap-3{column-gap:1rem !important}.column-gap-4{column-gap:1.5rem !important}.column-gap-5{column-gap:3rem !important}.font-monospace{font-family:var(--bs-font-monospace) !important}.fs-1{font-size:calc(1.325rem + 0.9vw) !important}.fs-2{font-size:calc(1.29rem + 0.48vw) !important}.fs-3{font-size:calc(1.27rem + 0.24vw) !important}.fs-4{font-size:1.25rem !important}.fs-5{font-size:1.1rem !important}.fs-6{font-size:1rem !important}.fst-italic{font-style:italic !important}.fst-normal{font-style:normal !important}.fw-lighter{font-weight:lighter !important}.fw-light{font-weight:300 !important}.fw-normal{font-weight:400 !important}.fw-medium{font-weight:500 !important}.fw-semibold{font-weight:600 !important}.fw-bold{font-weight:700 !important}.fw-bolder{font-weight:bolder !important}.lh-1{line-height:1 !important}.lh-sm{line-height:1.25 !important}.lh-base{line-height:1.5 !important}.lh-lg{line-height:2 !important}.text-start{text-align:left !important}.text-end{text-align:right !important}.text-center{text-align:center !important}.text-decoration-none{text-decoration:none !important}.text-decoration-underline{text-decoration:underline !important}.text-decoration-line-through{text-decoration:line-through !important}.text-lowercase{text-transform:lowercase !important}.text-uppercase{text-transform:uppercase !important}.text-capitalize{text-transform:capitalize !important}.text-wrap{white-space:normal !important}.text-nowrap{white-space:nowrap !important}.text-break{word-wrap:break-word !important;word-break:break-word !important}.text-default{--bs-text-opacity: 1;color:rgba(var(--bs-default-rgb), var(--bs-text-opacity)) !important}.text-primary{--bs-text-opacity: 1;color:rgba(var(--bs-primary-rgb), var(--bs-text-opacity)) !important}.text-secondary{--bs-text-opacity: 1;color:rgba(var(--bs-secondary-rgb), var(--bs-text-opacity)) !important}.text-success{--bs-text-opacity: 1;color:rgba(var(--bs-success-rgb), var(--bs-text-opacity)) !important}.text-info{--bs-text-opacity: 1;color:rgba(var(--bs-info-rgb), var(--bs-text-opacity)) !important}.text-warning{--bs-text-opacity: 1;color:rgba(var(--bs-warning-rgb), var(--bs-text-opacity)) !important}.text-danger{--bs-text-opacity: 1;color:rgba(var(--bs-danger-rgb), var(--bs-text-opacity)) !important}.text-light{--bs-text-opacity: 1;color:rgba(var(--bs-light-rgb), var(--bs-text-opacity)) !important}.text-dark{--bs-text-opacity: 1;color:rgba(var(--bs-dark-rgb), var(--bs-text-opacity)) !important}.text-black{--bs-text-opacity: 1;color:rgba(var(--bs-black-rgb), var(--bs-text-opacity)) !important}.text-white{--bs-text-opacity: 1;color:rgba(var(--bs-white-rgb), var(--bs-text-opacity)) !important}.text-body{--bs-text-opacity: 1;color:rgba(var(--bs-body-color-rgb), var(--bs-text-opacity)) !important}.text-muted{--bs-text-opacity: 1;color:var(--bs-secondary-color) !important}.text-black-50{--bs-text-opacity: 1;color:rgba(0,0,0,.5) !important}.text-white-50{--bs-text-opacity: 1;color:rgba(255,255,255,.5) !important}.text-body-secondary{--bs-text-opacity: 1;color:var(--bs-secondary-color) !important}.text-body-tertiary{--bs-text-opacity: 1;color:var(--bs-tertiary-color) !important}.text-body-emphasis{--bs-text-opacity: 1;color:var(--bs-emphasis-color) !important}.text-reset{--bs-text-opacity: 1;color:inherit !important}.text-opacity-25{--bs-text-opacity: 0.25}.text-opacity-50{--bs-text-opacity: 0.5}.text-opacity-75{--bs-text-opacity: 0.75}.text-opacity-100{--bs-text-opacity: 1}.text-primary-emphasis{color:var(--bs-primary-text-emphasis) !important}.text-secondary-emphasis{color:var(--bs-secondary-text-emphasis) !important}.text-success-emphasis{color:var(--bs-success-text-emphasis) !important}.text-info-emphasis{color:var(--bs-info-text-emphasis) !important}.text-warning-emphasis{color:var(--bs-warning-text-emphasis) !important}.text-danger-emphasis{color:var(--bs-danger-text-emphasis) !important}.text-light-emphasis{color:var(--bs-light-text-emphasis) !important}.text-dark-emphasis{color:var(--bs-dark-text-emphasis) !important}.link-opacity-10{--bs-link-opacity: 0.1}.link-opacity-10-hover:hover{--bs-link-opacity: 0.1}.link-opacity-25{--bs-link-opacity: 0.25}.link-opacity-25-hover:hover{--bs-link-opacity: 0.25}.link-opacity-50{--bs-link-opacity: 0.5}.link-opacity-50-hover:hover{--bs-link-opacity: 0.5}.link-opacity-75{--bs-link-opacity: 0.75}.link-opacity-75-hover:hover{--bs-link-opacity: 0.75}.link-opacity-100{--bs-link-opacity: 1}.link-opacity-100-hover:hover{--bs-link-opacity: 1}.link-offset-1{text-underline-offset:.125em !important}.link-offset-1-hover:hover{text-underline-offset:.125em !important}.link-offset-2{text-underline-offset:.25em !important}.link-offset-2-hover:hover{text-underline-offset:.25em !important}.link-offset-3{text-underline-offset:.375em !important}.link-offset-3-hover:hover{text-underline-offset:.375em !important}.link-underline-default{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-default-rgb), var(--bs-link-underline-opacity)) !important}.link-underline-primary{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-primary-rgb), var(--bs-link-underline-opacity)) !important}.link-underline-secondary{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-secondary-rgb), var(--bs-link-underline-opacity)) !important}.link-underline-success{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-success-rgb), var(--bs-link-underline-opacity)) !important}.link-underline-info{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-info-rgb), var(--bs-link-underline-opacity)) !important}.link-underline-warning{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-warning-rgb), var(--bs-link-underline-opacity)) !important}.link-underline-danger{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-danger-rgb), var(--bs-link-underline-opacity)) !important}.link-underline-light{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-light-rgb), var(--bs-link-underline-opacity)) !important}.link-underline-dark{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-dark-rgb), var(--bs-link-underline-opacity)) !important}.link-underline{--bs-link-underline-opacity: 1;text-decoration-color:rgba(var(--bs-link-color-rgb), var(--bs-link-underline-opacity, 1)) !important}.link-underline-opacity-0{--bs-link-underline-opacity: 0}.link-underline-opacity-0-hover:hover{--bs-link-underline-opacity: 0}.link-underline-opacity-10{--bs-link-underline-opacity: 0.1}.link-underline-opacity-10-hover:hover{--bs-link-underline-opacity: 0.1}.link-underline-opacity-25{--bs-link-underline-opacity: 0.25}.link-underline-opacity-25-hover:hover{--bs-link-underline-opacity: 0.25}.link-underline-opacity-50{--bs-link-underline-opacity: 0.5}.link-underline-opacity-50-hover:hover{--bs-link-underline-opacity: 0.5}.link-underline-opacity-75{--bs-link-underline-opacity: 0.75}.link-underline-opacity-75-hover:hover{--bs-link-underline-opacity: 0.75}.link-underline-opacity-100{--bs-link-underline-opacity: 1}.link-underline-opacity-100-hover:hover{--bs-link-underline-opacity: 1}.bg-default{--bs-bg-opacity: 1;background-color:rgba(var(--bs-default-rgb), var(--bs-bg-opacity)) !important}.bg-primary{--bs-bg-opacity: 1;background-color:rgba(var(--bs-primary-rgb), var(--bs-bg-opacity)) !important}.bg-secondary{--bs-bg-opacity: 1;background-color:rgba(var(--bs-secondary-rgb), var(--bs-bg-opacity)) !important}.bg-success{--bs-bg-opacity: 1;background-color:rgba(var(--bs-success-rgb), var(--bs-bg-opacity)) !important}.bg-info{--bs-bg-opacity: 1;background-color:rgba(var(--bs-info-rgb), var(--bs-bg-opacity)) !important}.bg-warning{--bs-bg-opacity: 1;background-color:rgba(var(--bs-warning-rgb), var(--bs-bg-opacity)) !important}.bg-danger{--bs-bg-opacity: 1;background-color:rgba(var(--bs-danger-rgb), var(--bs-bg-opacity)) !important}.bg-light{--bs-bg-opacity: 1;background-color:rgba(var(--bs-light-rgb), var(--bs-bg-opacity)) !important}.bg-dark{--bs-bg-opacity: 1;background-color:rgba(var(--bs-dark-rgb), var(--bs-bg-opacity)) !important}.bg-black{--bs-bg-opacity: 1;background-color:rgba(var(--bs-black-rgb), var(--bs-bg-opacity)) !important}.bg-white{--bs-bg-opacity: 1;background-color:rgba(var(--bs-white-rgb), var(--bs-bg-opacity)) !important}.bg-body{--bs-bg-opacity: 1;background-color:rgba(var(--bs-body-bg-rgb), var(--bs-bg-opacity)) !important}.bg-transparent{--bs-bg-opacity: 1;background-color:rgba(0,0,0,0) !important}.bg-body-secondary{--bs-bg-opacity: 1;background-color:rgba(var(--bs-secondary-bg-rgb), var(--bs-bg-opacity)) !important}.bg-body-tertiary{--bs-bg-opacity: 1;background-color:rgba(var(--bs-tertiary-bg-rgb), var(--bs-bg-opacity)) !important}.bg-opacity-10{--bs-bg-opacity: 0.1}.bg-opacity-25{--bs-bg-opacity: 0.25}.bg-opacity-50{--bs-bg-opacity: 0.5}.bg-opacity-75{--bs-bg-opacity: 0.75}.bg-opacity-100{--bs-bg-opacity: 1}.bg-primary-subtle{background-color:var(--bs-primary-bg-subtle) !important}.bg-secondary-subtle{background-color:var(--bs-secondary-bg-subtle) !important}.bg-success-subtle{background-color:var(--bs-success-bg-subtle) !important}.bg-info-subtle{background-color:var(--bs-info-bg-subtle) !important}.bg-warning-subtle{background-color:var(--bs-warning-bg-subtle) !important}.bg-danger-subtle{background-color:var(--bs-danger-bg-subtle) !important}.bg-light-subtle{background-color:var(--bs-light-bg-subtle) !important}.bg-dark-subtle{background-color:var(--bs-dark-bg-subtle) !important}.bg-gradient{background-image:var(--bs-gradient) !important}.user-select-all{user-select:all !important}.user-select-auto{user-select:auto !important}.user-select-none{user-select:none !important}.pe-none{pointer-events:none !important}.pe-auto{pointer-events:auto !important}.rounded{border-radius:var(--bs-border-radius) !important}.rounded-0{border-radius:0 !important}.rounded-1{border-radius:var(--bs-border-radius-sm) !important}.rounded-2{border-radius:var(--bs-border-radius) !important}.rounded-3{border-radius:var(--bs-border-radius-lg) !important}.rounded-4{border-radius:var(--bs-border-radius-xl) !important}.rounded-5{border-radius:var(--bs-border-radius-xxl) !important}.rounded-circle{border-radius:50% !important}.rounded-pill{border-radius:var(--bs-border-radius-pill) !important}.rounded-top{border-top-left-radius:var(--bs-border-radius) !important;border-top-right-radius:var(--bs-border-radius) !important}.rounded-top-0{border-top-left-radius:0 !important;border-top-right-radius:0 !important}.rounded-top-1{border-top-left-radius:var(--bs-border-radius-sm) !important;border-top-right-radius:var(--bs-border-radius-sm) !important}.rounded-top-2{border-top-left-radius:var(--bs-border-radius) !important;border-top-right-radius:var(--bs-border-radius) !important}.rounded-top-3{border-top-left-radius:var(--bs-border-radius-lg) !important;border-top-right-radius:var(--bs-border-radius-lg) !important}.rounded-top-4{border-top-left-radius:var(--bs-border-radius-xl) !important;border-top-right-radius:var(--bs-border-radius-xl) !important}.rounded-top-5{border-top-left-radius:var(--bs-border-radius-xxl) !important;border-top-right-radius:var(--bs-border-radius-xxl) !important}.rounded-top-circle{border-top-left-radius:50% !important;border-top-right-radius:50% !important}.rounded-top-pill{border-top-left-radius:var(--bs-border-radius-pill) !important;border-top-right-radius:var(--bs-border-radius-pill) !important}.rounded-end{border-top-right-radius:var(--bs-border-radius) !important;border-bottom-right-radius:var(--bs-border-radius) !important}.rounded-end-0{border-top-right-radius:0 !important;border-bottom-right-radius:0 !important}.rounded-end-1{border-top-right-radius:var(--bs-border-radius-sm) !important;border-bottom-right-radius:var(--bs-border-radius-sm) !important}.rounded-end-2{border-top-right-radius:var(--bs-border-radius) !important;border-bottom-right-radius:var(--bs-border-radius) !important}.rounded-end-3{border-top-right-radius:var(--bs-border-radius-lg) !important;border-bottom-right-radius:var(--bs-border-radius-lg) !important}.rounded-end-4{border-top-right-radius:var(--bs-border-radius-xl) !important;border-bottom-right-radius:var(--bs-border-radius-xl) !important}.rounded-end-5{border-top-right-radius:var(--bs-border-radius-xxl) !important;border-bottom-right-radius:var(--bs-border-radius-xxl) !important}.rounded-end-circle{border-top-right-radius:50% !important;border-bottom-right-radius:50% !important}.rounded-end-pill{border-top-right-radius:var(--bs-border-radius-pill) !important;border-bottom-right-radius:var(--bs-border-radius-pill) !important}.rounded-bottom{border-bottom-right-radius:var(--bs-border-radius) !important;border-bottom-left-radius:var(--bs-border-radius) !important}.rounded-bottom-0{border-bottom-right-radius:0 !important;border-bottom-left-radius:0 !important}.rounded-bottom-1{border-bottom-right-radius:var(--bs-border-radius-sm) !important;border-bottom-left-radius:var(--bs-border-radius-sm) !important}.rounded-bottom-2{border-bottom-right-radius:var(--bs-border-radius) !important;border-bottom-left-radius:var(--bs-border-radius) !important}.rounded-bottom-3{border-bottom-right-radius:var(--bs-border-radius-lg) !important;border-bottom-left-radius:var(--bs-border-radius-lg) !important}.rounded-bottom-4{border-bottom-right-radius:var(--bs-border-radius-xl) !important;border-bottom-left-radius:var(--bs-border-radius-xl) !important}.rounded-bottom-5{border-bottom-right-radius:var(--bs-border-radius-xxl) !important;border-bottom-left-radius:var(--bs-border-radius-xxl) !important}.rounded-bottom-circle{border-bottom-right-radius:50% !important;border-bottom-left-radius:50% !important}.rounded-bottom-pill{border-bottom-right-radius:var(--bs-border-radius-pill) !important;border-bottom-left-radius:var(--bs-border-radius-pill) !important}.rounded-start{border-bottom-left-radius:var(--bs-border-radius) !important;border-top-left-radius:var(--bs-border-radius) !important}.rounded-start-0{border-bottom-left-radius:0 !important;border-top-left-radius:0 !important}.rounded-start-1{border-bottom-left-radius:var(--bs-border-radius-sm) !important;border-top-left-radius:var(--bs-border-radius-sm) !important}.rounded-start-2{border-bottom-left-radius:var(--bs-border-radius) !important;border-top-left-radius:var(--bs-border-radius) !important}.rounded-start-3{border-bottom-left-radius:var(--bs-border-radius-lg) !important;border-top-left-radius:var(--bs-border-radius-lg) !important}.rounded-start-4{border-bottom-left-radius:var(--bs-border-radius-xl) !important;border-top-left-radius:var(--bs-border-radius-xl) !important}.rounded-start-5{border-bottom-left-radius:var(--bs-border-radius-xxl) !important;border-top-left-radius:var(--bs-border-radius-xxl) !important}.rounded-start-circle{border-bottom-left-radius:50% !important;border-top-left-radius:50% !important}.rounded-start-pill{border-bottom-left-radius:var(--bs-border-radius-pill) !important;border-top-left-radius:var(--bs-border-radius-pill) !important}.visible{visibility:visible !important}.invisible{visibility:hidden !important}.z-n1{z-index:-1 !important}.z-0{z-index:0 !important}.z-1{z-index:1 !important}.z-2{z-index:2 !important}.z-3{z-index:3 !important}@media(min-width: 576px){.float-sm-start{float:left !important}.float-sm-end{float:right !important}.float-sm-none{float:none !important}.object-fit-sm-contain{object-fit:contain !important}.object-fit-sm-cover{object-fit:cover !important}.object-fit-sm-fill{object-fit:fill !important}.object-fit-sm-scale{object-fit:scale-down !important}.object-fit-sm-none{object-fit:none !important}.d-sm-inline{display:inline !important}.d-sm-inline-block{display:inline-block !important}.d-sm-block{display:block !important}.d-sm-grid{display:grid !important}.d-sm-inline-grid{display:inline-grid !important}.d-sm-table{display:table !important}.d-sm-table-row{display:table-row !important}.d-sm-table-cell{display:table-cell !important}.d-sm-flex{display:flex !important}.d-sm-inline-flex{display:inline-flex !important}.d-sm-none{display:none !important}.flex-sm-fill{flex:1 1 auto !important}.flex-sm-row{flex-direction:row !important}.flex-sm-column{flex-direction:column !important}.flex-sm-row-reverse{flex-direction:row-reverse !important}.flex-sm-column-reverse{flex-direction:column-reverse !important}.flex-sm-grow-0{flex-grow:0 !important}.flex-sm-grow-1{flex-grow:1 !important}.flex-sm-shrink-0{flex-shrink:0 !important}.flex-sm-shrink-1{flex-shrink:1 !important}.flex-sm-wrap{flex-wrap:wrap !important}.flex-sm-nowrap{flex-wrap:nowrap !important}.flex-sm-wrap-reverse{flex-wrap:wrap-reverse !important}.justify-content-sm-start{justify-content:flex-start !important}.justify-content-sm-end{justify-content:flex-end !important}.justify-content-sm-center{justify-content:center !important}.justify-content-sm-between{justify-content:space-between !important}.justify-content-sm-around{justify-content:space-around !important}.justify-content-sm-evenly{justify-content:space-evenly !important}.align-items-sm-start{align-items:flex-start !important}.align-items-sm-end{align-items:flex-end !important}.align-items-sm-center{align-items:center !important}.align-items-sm-baseline{align-items:baseline !important}.align-items-sm-stretch{align-items:stretch !important}.align-content-sm-start{align-content:flex-start !important}.align-content-sm-end{align-content:flex-end !important}.align-content-sm-center{align-content:center !important}.align-content-sm-between{align-content:space-between !important}.align-content-sm-around{align-content:space-around !important}.align-content-sm-stretch{align-content:stretch !important}.align-self-sm-auto{align-self:auto !important}.align-self-sm-start{align-self:flex-start !important}.align-self-sm-end{align-self:flex-end !important}.align-self-sm-center{align-self:center !important}.align-self-sm-baseline{align-self:baseline !important}.align-self-sm-stretch{align-self:stretch !important}.order-sm-first{order:-1 !important}.order-sm-0{order:0 !important}.order-sm-1{order:1 !important}.order-sm-2{order:2 !important}.order-sm-3{order:3 !important}.order-sm-4{order:4 !important}.order-sm-5{order:5 !important}.order-sm-last{order:6 !important}.m-sm-0{margin:0 !important}.m-sm-1{margin:.25rem !important}.m-sm-2{margin:.5rem !important}.m-sm-3{margin:1rem !important}.m-sm-4{margin:1.5rem !important}.m-sm-5{margin:3rem !important}.m-sm-auto{margin:auto !important}.mx-sm-0{margin-right:0 !important;margin-left:0 !important}.mx-sm-1{margin-right:.25rem !important;margin-left:.25rem !important}.mx-sm-2{margin-right:.5rem !important;margin-left:.5rem !important}.mx-sm-3{margin-right:1rem !important;margin-left:1rem !important}.mx-sm-4{margin-right:1.5rem !important;margin-left:1.5rem !important}.mx-sm-5{margin-right:3rem !important;margin-left:3rem !important}.mx-sm-auto{margin-right:auto !important;margin-left:auto !important}.my-sm-0{margin-top:0 !important;margin-bottom:0 !important}.my-sm-1{margin-top:.25rem !important;margin-bottom:.25rem !important}.my-sm-2{margin-top:.5rem !important;margin-bottom:.5rem !important}.my-sm-3{margin-top:1rem !important;margin-bottom:1rem !important}.my-sm-4{margin-top:1.5rem !important;margin-bottom:1.5rem !important}.my-sm-5{margin-top:3rem !important;margin-bottom:3rem !important}.my-sm-auto{margin-top:auto !important;margin-bottom:auto !important}.mt-sm-0{margin-top:0 !important}.mt-sm-1{margin-top:.25rem !important}.mt-sm-2{margin-top:.5rem !important}.mt-sm-3{margin-top:1rem !important}.mt-sm-4{margin-top:1.5rem !important}.mt-sm-5{margin-top:3rem !important}.mt-sm-auto{margin-top:auto !important}.me-sm-0{margin-right:0 !important}.me-sm-1{margin-right:.25rem !important}.me-sm-2{margin-right:.5rem !important}.me-sm-3{margin-right:1rem !important}.me-sm-4{margin-right:1.5rem !important}.me-sm-5{margin-right:3rem !important}.me-sm-auto{margin-right:auto !important}.mb-sm-0{margin-bottom:0 !important}.mb-sm-1{margin-bottom:.25rem !important}.mb-sm-2{margin-bottom:.5rem !important}.mb-sm-3{margin-bottom:1rem !important}.mb-sm-4{margin-bottom:1.5rem !important}.mb-sm-5{margin-bottom:3rem !important}.mb-sm-auto{margin-bottom:auto !important}.ms-sm-0{margin-left:0 !important}.ms-sm-1{margin-left:.25rem !important}.ms-sm-2{margin-left:.5rem !important}.ms-sm-3{margin-left:1rem !important}.ms-sm-4{margin-left:1.5rem !important}.ms-sm-5{margin-left:3rem !important}.ms-sm-auto{margin-left:auto !important}.p-sm-0{padding:0 !important}.p-sm-1{padding:.25rem !important}.p-sm-2{padding:.5rem !important}.p-sm-3{padding:1rem !important}.p-sm-4{padding:1.5rem !important}.p-sm-5{padding:3rem !important}.px-sm-0{padding-right:0 !important;padding-left:0 !important}.px-sm-1{padding-right:.25rem !important;padding-left:.25rem !important}.px-sm-2{padding-right:.5rem !important;padding-left:.5rem !important}.px-sm-3{padding-right:1rem !important;padding-left:1rem !important}.px-sm-4{padding-right:1.5rem !important;padding-left:1.5rem !important}.px-sm-5{padding-right:3rem !important;padding-left:3rem !important}.py-sm-0{padding-top:0 !important;padding-bottom:0 !important}.py-sm-1{padding-top:.25rem !important;padding-bottom:.25rem !important}.py-sm-2{padding-top:.5rem !important;padding-bottom:.5rem !important}.py-sm-3{padding-top:1rem !important;padding-bottom:1rem !important}.py-sm-4{padding-top:1.5rem !important;padding-bottom:1.5rem !important}.py-sm-5{padding-top:3rem !important;padding-bottom:3rem !important}.pt-sm-0{padding-top:0 !important}.pt-sm-1{padding-top:.25rem !important}.pt-sm-2{padding-top:.5rem !important}.pt-sm-3{padding-top:1rem !important}.pt-sm-4{padding-top:1.5rem !important}.pt-sm-5{padding-top:3rem !important}.pe-sm-0{padding-right:0 !important}.pe-sm-1{padding-right:.25rem !important}.pe-sm-2{padding-right:.5rem !important}.pe-sm-3{padding-right:1rem !important}.pe-sm-4{padding-right:1.5rem !important}.pe-sm-5{padding-right:3rem !important}.pb-sm-0{padding-bottom:0 !important}.pb-sm-1{padding-bottom:.25rem !important}.pb-sm-2{padding-bottom:.5rem !important}.pb-sm-3{padding-bottom:1rem !important}.pb-sm-4{padding-bottom:1.5rem !important}.pb-sm-5{padding-bottom:3rem !important}.ps-sm-0{padding-left:0 !important}.ps-sm-1{padding-left:.25rem !important}.ps-sm-2{padding-left:.5rem !important}.ps-sm-3{padding-left:1rem !important}.ps-sm-4{padding-left:1.5rem !important}.ps-sm-5{padding-left:3rem !important}.gap-sm-0{gap:0 !important}.gap-sm-1{gap:.25rem !important}.gap-sm-2{gap:.5rem !important}.gap-sm-3{gap:1rem !important}.gap-sm-4{gap:1.5rem !important}.gap-sm-5{gap:3rem !important}.row-gap-sm-0{row-gap:0 !important}.row-gap-sm-1{row-gap:.25rem !important}.row-gap-sm-2{row-gap:.5rem !important}.row-gap-sm-3{row-gap:1rem !important}.row-gap-sm-4{row-gap:1.5rem !important}.row-gap-sm-5{row-gap:3rem !important}.column-gap-sm-0{column-gap:0 !important}.column-gap-sm-1{column-gap:.25rem !important}.column-gap-sm-2{column-gap:.5rem !important}.column-gap-sm-3{column-gap:1rem !important}.column-gap-sm-4{column-gap:1.5rem !important}.column-gap-sm-5{column-gap:3rem !important}.text-sm-start{text-align:left !important}.text-sm-end{text-align:right !important}.text-sm-center{text-align:center !important}}@media(min-width: 768px){.float-md-start{float:left !important}.float-md-end{float:right !important}.float-md-none{float:none !important}.object-fit-md-contain{object-fit:contain !important}.object-fit-md-cover{object-fit:cover !important}.object-fit-md-fill{object-fit:fill !important}.object-fit-md-scale{object-fit:scale-down !important}.object-fit-md-none{object-fit:none !important}.d-md-inline{display:inline !important}.d-md-inline-block{display:inline-block !important}.d-md-block{display:block !important}.d-md-grid{display:grid !important}.d-md-inline-grid{display:inline-grid !important}.d-md-table{display:table !important}.d-md-table-row{display:table-row !important}.d-md-table-cell{display:table-cell !important}.d-md-flex{display:flex !important}.d-md-inline-flex{display:inline-flex !important}.d-md-none{display:none !important}.flex-md-fill{flex:1 1 auto !important}.flex-md-row{flex-direction:row !important}.flex-md-column{flex-direction:column !important}.flex-md-row-reverse{flex-direction:row-reverse !important}.flex-md-column-reverse{flex-direction:column-reverse !important}.flex-md-grow-0{flex-grow:0 !important}.flex-md-grow-1{flex-grow:1 !important}.flex-md-shrink-0{flex-shrink:0 !important}.flex-md-shrink-1{flex-shrink:1 !important}.flex-md-wrap{flex-wrap:wrap !important}.flex-md-nowrap{flex-wrap:nowrap !important}.flex-md-wrap-reverse{flex-wrap:wrap-reverse !important}.justify-content-md-start{justify-content:flex-start !important}.justify-content-md-end{justify-content:flex-end !important}.justify-content-md-center{justify-content:center !important}.justify-content-md-between{justify-content:space-between !important}.justify-content-md-around{justify-content:space-around !important}.justify-content-md-evenly{justify-content:space-evenly !important}.align-items-md-start{align-items:flex-start !important}.align-items-md-end{align-items:flex-end !important}.align-items-md-center{align-items:center !important}.align-items-md-baseline{align-items:baseline !important}.align-items-md-stretch{align-items:stretch !important}.align-content-md-start{align-content:flex-start !important}.align-content-md-end{align-content:flex-end !important}.align-content-md-center{align-content:center !important}.align-content-md-between{align-content:space-between !important}.align-content-md-around{align-content:space-around !important}.align-content-md-stretch{align-content:stretch !important}.align-self-md-auto{align-self:auto !important}.align-self-md-start{align-self:flex-start !important}.align-self-md-end{align-self:flex-end !important}.align-self-md-center{align-self:center !important}.align-self-md-baseline{align-self:baseline !important}.align-self-md-stretch{align-self:stretch !important}.order-md-first{order:-1 !important}.order-md-0{order:0 !important}.order-md-1{order:1 !important}.order-md-2{order:2 !important}.order-md-3{order:3 !important}.order-md-4{order:4 !important}.order-md-5{order:5 !important}.order-md-last{order:6 !important}.m-md-0{margin:0 !important}.m-md-1{margin:.25rem !important}.m-md-2{margin:.5rem !important}.m-md-3{margin:1rem !important}.m-md-4{margin:1.5rem !important}.m-md-5{margin:3rem !important}.m-md-auto{margin:auto !important}.mx-md-0{margin-right:0 !important;margin-left:0 !important}.mx-md-1{margin-right:.25rem !important;margin-left:.25rem !important}.mx-md-2{margin-right:.5rem !important;margin-left:.5rem !important}.mx-md-3{margin-right:1rem !important;margin-left:1rem !important}.mx-md-4{margin-right:1.5rem !important;margin-left:1.5rem !important}.mx-md-5{margin-right:3rem !important;margin-left:3rem !important}.mx-md-auto{margin-right:auto !important;margin-left:auto !important}.my-md-0{margin-top:0 !important;margin-bottom:0 !important}.my-md-1{margin-top:.25rem !important;margin-bottom:.25rem !important}.my-md-2{margin-top:.5rem !important;margin-bottom:.5rem !important}.my-md-3{margin-top:1rem !important;margin-bottom:1rem !important}.my-md-4{margin-top:1.5rem !important;margin-bottom:1.5rem !important}.my-md-5{margin-top:3rem !important;margin-bottom:3rem !important}.my-md-auto{margin-top:auto !important;margin-bottom:auto !important}.mt-md-0{margin-top:0 !important}.mt-md-1{margin-top:.25rem !important}.mt-md-2{margin-top:.5rem !important}.mt-md-3{margin-top:1rem !important}.mt-md-4{margin-top:1.5rem !important}.mt-md-5{margin-top:3rem !important}.mt-md-auto{margin-top:auto !important}.me-md-0{margin-right:0 !important}.me-md-1{margin-right:.25rem !important}.me-md-2{margin-right:.5rem !important}.me-md-3{margin-right:1rem !important}.me-md-4{margin-right:1.5rem !important}.me-md-5{margin-right:3rem !important}.me-md-auto{margin-right:auto !important}.mb-md-0{margin-bottom:0 !important}.mb-md-1{margin-bottom:.25rem !important}.mb-md-2{margin-bottom:.5rem !important}.mb-md-3{margin-bottom:1rem !important}.mb-md-4{margin-bottom:1.5rem !important}.mb-md-5{margin-bottom:3rem !important}.mb-md-auto{margin-bottom:auto !important}.ms-md-0{margin-left:0 !important}.ms-md-1{margin-left:.25rem !important}.ms-md-2{margin-left:.5rem !important}.ms-md-3{margin-left:1rem !important}.ms-md-4{margin-left:1.5rem !important}.ms-md-5{margin-left:3rem !important}.ms-md-auto{margin-left:auto !important}.p-md-0{padding:0 !important}.p-md-1{padding:.25rem !important}.p-md-2{padding:.5rem !important}.p-md-3{padding:1rem !important}.p-md-4{padding:1.5rem !important}.p-md-5{padding:3rem !important}.px-md-0{padding-right:0 !important;padding-left:0 !important}.px-md-1{padding-right:.25rem !important;padding-left:.25rem !important}.px-md-2{padding-right:.5rem !important;padding-left:.5rem !important}.px-md-3{padding-right:1rem !important;padding-left:1rem !important}.px-md-4{padding-right:1.5rem !important;padding-left:1.5rem !important}.px-md-5{padding-right:3rem !important;padding-left:3rem !important}.py-md-0{padding-top:0 !important;padding-bottom:0 !important}.py-md-1{padding-top:.25rem !important;padding-bottom:.25rem !important}.py-md-2{padding-top:.5rem !important;padding-bottom:.5rem !important}.py-md-3{padding-top:1rem !important;padding-bottom:1rem !important}.py-md-4{padding-top:1.5rem !important;padding-bottom:1.5rem !important}.py-md-5{padding-top:3rem !important;padding-bottom:3rem !important}.pt-md-0{padding-top:0 !important}.pt-md-1{padding-top:.25rem !important}.pt-md-2{padding-top:.5rem !important}.pt-md-3{padding-top:1rem !important}.pt-md-4{padding-top:1.5rem !important}.pt-md-5{padding-top:3rem !important}.pe-md-0{padding-right:0 !important}.pe-md-1{padding-right:.25rem !important}.pe-md-2{padding-right:.5rem !important}.pe-md-3{padding-right:1rem !important}.pe-md-4{padding-right:1.5rem !important}.pe-md-5{padding-right:3rem !important}.pb-md-0{padding-bottom:0 !important}.pb-md-1{padding-bottom:.25rem !important}.pb-md-2{padding-bottom:.5rem !important}.pb-md-3{padding-bottom:1rem !important}.pb-md-4{padding-bottom:1.5rem !important}.pb-md-5{padding-bottom:3rem !important}.ps-md-0{padding-left:0 !important}.ps-md-1{padding-left:.25rem !important}.ps-md-2{padding-left:.5rem !important}.ps-md-3{padding-left:1rem !important}.ps-md-4{padding-left:1.5rem !important}.ps-md-5{padding-left:3rem !important}.gap-md-0{gap:0 !important}.gap-md-1{gap:.25rem !important}.gap-md-2{gap:.5rem !important}.gap-md-3{gap:1rem !important}.gap-md-4{gap:1.5rem !important}.gap-md-5{gap:3rem !important}.row-gap-md-0{row-gap:0 !important}.row-gap-md-1{row-gap:.25rem !important}.row-gap-md-2{row-gap:.5rem !important}.row-gap-md-3{row-gap:1rem !important}.row-gap-md-4{row-gap:1.5rem !important}.row-gap-md-5{row-gap:3rem !important}.column-gap-md-0{column-gap:0 !important}.column-gap-md-1{column-gap:.25rem !important}.column-gap-md-2{column-gap:.5rem !important}.column-gap-md-3{column-gap:1rem !important}.column-gap-md-4{column-gap:1.5rem !important}.column-gap-md-5{column-gap:3rem !important}.text-md-start{text-align:left !important}.text-md-end{text-align:right !important}.text-md-center{text-align:center !important}}@media(min-width: 992px){.float-lg-start{float:left !important}.float-lg-end{float:right !important}.float-lg-none{float:none !important}.object-fit-lg-contain{object-fit:contain !important}.object-fit-lg-cover{object-fit:cover !important}.object-fit-lg-fill{object-fit:fill !important}.object-fit-lg-scale{object-fit:scale-down !important}.object-fit-lg-none{object-fit:none !important}.d-lg-inline{display:inline !important}.d-lg-inline-block{display:inline-block !important}.d-lg-block{display:block !important}.d-lg-grid{display:grid !important}.d-lg-inline-grid{display:inline-grid !important}.d-lg-table{display:table !important}.d-lg-table-row{display:table-row !important}.d-lg-table-cell{display:table-cell !important}.d-lg-flex{display:flex !important}.d-lg-inline-flex{display:inline-flex !important}.d-lg-none{display:none !important}.flex-lg-fill{flex:1 1 auto !important}.flex-lg-row{flex-direction:row !important}.flex-lg-column{flex-direction:column !important}.flex-lg-row-reverse{flex-direction:row-reverse !important}.flex-lg-column-reverse{flex-direction:column-reverse !important}.flex-lg-grow-0{flex-grow:0 !important}.flex-lg-grow-1{flex-grow:1 !important}.flex-lg-shrink-0{flex-shrink:0 !important}.flex-lg-shrink-1{flex-shrink:1 !important}.flex-lg-wrap{flex-wrap:wrap !important}.flex-lg-nowrap{flex-wrap:nowrap !important}.flex-lg-wrap-reverse{flex-wrap:wrap-reverse !important}.justify-content-lg-start{justify-content:flex-start !important}.justify-content-lg-end{justify-content:flex-end !important}.justify-content-lg-center{justify-content:center !important}.justify-content-lg-between{justify-content:space-between !important}.justify-content-lg-around{justify-content:space-around !important}.justify-content-lg-evenly{justify-content:space-evenly !important}.align-items-lg-start{align-items:flex-start !important}.align-items-lg-end{align-items:flex-end !important}.align-items-lg-center{align-items:center !important}.align-items-lg-baseline{align-items:baseline !important}.align-items-lg-stretch{align-items:stretch !important}.align-content-lg-start{align-content:flex-start !important}.align-content-lg-end{align-content:flex-end !important}.align-content-lg-center{align-content:center !important}.align-content-lg-between{align-content:space-between !important}.align-content-lg-around{align-content:space-around !important}.align-content-lg-stretch{align-content:stretch !important}.align-self-lg-auto{align-self:auto !important}.align-self-lg-start{align-self:flex-start !important}.align-self-lg-end{align-self:flex-end !important}.align-self-lg-center{align-self:center !important}.align-self-lg-baseline{align-self:baseline !important}.align-self-lg-stretch{align-self:stretch !important}.order-lg-first{order:-1 !important}.order-lg-0{order:0 !important}.order-lg-1{order:1 !important}.order-lg-2{order:2 !important}.order-lg-3{order:3 !important}.order-lg-4{order:4 !important}.order-lg-5{order:5 !important}.order-lg-last{order:6 !important}.m-lg-0{margin:0 !important}.m-lg-1{margin:.25rem !important}.m-lg-2{margin:.5rem !important}.m-lg-3{margin:1rem !important}.m-lg-4{margin:1.5rem !important}.m-lg-5{margin:3rem !important}.m-lg-auto{margin:auto !important}.mx-lg-0{margin-right:0 !important;margin-left:0 !important}.mx-lg-1{margin-right:.25rem !important;margin-left:.25rem !important}.mx-lg-2{margin-right:.5rem !important;margin-left:.5rem !important}.mx-lg-3{margin-right:1rem !important;margin-left:1rem !important}.mx-lg-4{margin-right:1.5rem !important;margin-left:1.5rem !important}.mx-lg-5{margin-right:3rem !important;margin-left:3rem !important}.mx-lg-auto{margin-right:auto !important;margin-left:auto !important}.my-lg-0{margin-top:0 !important;margin-bottom:0 !important}.my-lg-1{margin-top:.25rem !important;margin-bottom:.25rem !important}.my-lg-2{margin-top:.5rem !important;margin-bottom:.5rem !important}.my-lg-3{margin-top:1rem !important;margin-bottom:1rem !important}.my-lg-4{margin-top:1.5rem !important;margin-bottom:1.5rem !important}.my-lg-5{margin-top:3rem !important;margin-bottom:3rem !important}.my-lg-auto{margin-top:auto !important;margin-bottom:auto !important}.mt-lg-0{margin-top:0 !important}.mt-lg-1{margin-top:.25rem !important}.mt-lg-2{margin-top:.5rem !important}.mt-lg-3{margin-top:1rem !important}.mt-lg-4{margin-top:1.5rem !important}.mt-lg-5{margin-top:3rem !important}.mt-lg-auto{margin-top:auto !important}.me-lg-0{margin-right:0 !important}.me-lg-1{margin-right:.25rem !important}.me-lg-2{margin-right:.5rem !important}.me-lg-3{margin-right:1rem !important}.me-lg-4{margin-right:1.5rem !important}.me-lg-5{margin-right:3rem !important}.me-lg-auto{margin-right:auto !important}.mb-lg-0{margin-bottom:0 !important}.mb-lg-1{margin-bottom:.25rem !important}.mb-lg-2{margin-bottom:.5rem !important}.mb-lg-3{margin-bottom:1rem !important}.mb-lg-4{margin-bottom:1.5rem !important}.mb-lg-5{margin-bottom:3rem !important}.mb-lg-auto{margin-bottom:auto !important}.ms-lg-0{margin-left:0 !important}.ms-lg-1{margin-left:.25rem !important}.ms-lg-2{margin-left:.5rem !important}.ms-lg-3{margin-left:1rem !important}.ms-lg-4{margin-left:1.5rem !important}.ms-lg-5{margin-left:3rem !important}.ms-lg-auto{margin-left:auto !important}.p-lg-0{padding:0 !important}.p-lg-1{padding:.25rem !important}.p-lg-2{padding:.5rem !important}.p-lg-3{padding:1rem !important}.p-lg-4{padding:1.5rem !important}.p-lg-5{padding:3rem !important}.px-lg-0{padding-right:0 !important;padding-left:0 !important}.px-lg-1{padding-right:.25rem !important;padding-left:.25rem !important}.px-lg-2{padding-right:.5rem !important;padding-left:.5rem !important}.px-lg-3{padding-right:1rem !important;padding-left:1rem !important}.px-lg-4{padding-right:1.5rem !important;padding-left:1.5rem !important}.px-lg-5{padding-right:3rem !important;padding-left:3rem !important}.py-lg-0{padding-top:0 !important;padding-bottom:0 !important}.py-lg-1{padding-top:.25rem !important;padding-bottom:.25rem !important}.py-lg-2{padding-top:.5rem !important;padding-bottom:.5rem !important}.py-lg-3{padding-top:1rem !important;padding-bottom:1rem !important}.py-lg-4{padding-top:1.5rem !important;padding-bottom:1.5rem !important}.py-lg-5{padding-top:3rem !important;padding-bottom:3rem !important}.pt-lg-0{padding-top:0 !important}.pt-lg-1{padding-top:.25rem !important}.pt-lg-2{padding-top:.5rem !important}.pt-lg-3{padding-top:1rem !important}.pt-lg-4{padding-top:1.5rem !important}.pt-lg-5{padding-top:3rem !important}.pe-lg-0{padding-right:0 !important}.pe-lg-1{padding-right:.25rem !important}.pe-lg-2{padding-right:.5rem !important}.pe-lg-3{padding-right:1rem !important}.pe-lg-4{padding-right:1.5rem !important}.pe-lg-5{padding-right:3rem !important}.pb-lg-0{padding-bottom:0 !important}.pb-lg-1{padding-bottom:.25rem !important}.pb-lg-2{padding-bottom:.5rem !important}.pb-lg-3{padding-bottom:1rem !important}.pb-lg-4{padding-bottom:1.5rem !important}.pb-lg-5{padding-bottom:3rem !important}.ps-lg-0{padding-left:0 !important}.ps-lg-1{padding-left:.25rem !important}.ps-lg-2{padding-left:.5rem !important}.ps-lg-3{padding-left:1rem !important}.ps-lg-4{padding-left:1.5rem !important}.ps-lg-5{padding-left:3rem !important}.gap-lg-0{gap:0 !important}.gap-lg-1{gap:.25rem !important}.gap-lg-2{gap:.5rem !important}.gap-lg-3{gap:1rem !important}.gap-lg-4{gap:1.5rem !important}.gap-lg-5{gap:3rem !important}.row-gap-lg-0{row-gap:0 !important}.row-gap-lg-1{row-gap:.25rem !important}.row-gap-lg-2{row-gap:.5rem !important}.row-gap-lg-3{row-gap:1rem !important}.row-gap-lg-4{row-gap:1.5rem !important}.row-gap-lg-5{row-gap:3rem !important}.column-gap-lg-0{column-gap:0 !important}.column-gap-lg-1{column-gap:.25rem !important}.column-gap-lg-2{column-gap:.5rem !important}.column-gap-lg-3{column-gap:1rem !important}.column-gap-lg-4{column-gap:1.5rem !important}.column-gap-lg-5{column-gap:3rem !important}.text-lg-start{text-align:left !important}.text-lg-end{text-align:right !important}.text-lg-center{text-align:center !important}}@media(min-width: 1200px){.float-xl-start{float:left !important}.float-xl-end{float:right !important}.float-xl-none{float:none !important}.object-fit-xl-contain{object-fit:contain !important}.object-fit-xl-cover{object-fit:cover !important}.object-fit-xl-fill{object-fit:fill !important}.object-fit-xl-scale{object-fit:scale-down !important}.object-fit-xl-none{object-fit:none !important}.d-xl-inline{display:inline !important}.d-xl-inline-block{display:inline-block !important}.d-xl-block{display:block !important}.d-xl-grid{display:grid !important}.d-xl-inline-grid{display:inline-grid !important}.d-xl-table{display:table !important}.d-xl-table-row{display:table-row !important}.d-xl-table-cell{display:table-cell !important}.d-xl-flex{display:flex !important}.d-xl-inline-flex{display:inline-flex !important}.d-xl-none{display:none !important}.flex-xl-fill{flex:1 1 auto !important}.flex-xl-row{flex-direction:row !important}.flex-xl-column{flex-direction:column !important}.flex-xl-row-reverse{flex-direction:row-reverse !important}.flex-xl-column-reverse{flex-direction:column-reverse !important}.flex-xl-grow-0{flex-grow:0 !important}.flex-xl-grow-1{flex-grow:1 !important}.flex-xl-shrink-0{flex-shrink:0 !important}.flex-xl-shrink-1{flex-shrink:1 !important}.flex-xl-wrap{flex-wrap:wrap !important}.flex-xl-nowrap{flex-wrap:nowrap !important}.flex-xl-wrap-reverse{flex-wrap:wrap-reverse !important}.justify-content-xl-start{justify-content:flex-start !important}.justify-content-xl-end{justify-content:flex-end !important}.justify-content-xl-center{justify-content:center !important}.justify-content-xl-between{justify-content:space-between !important}.justify-content-xl-around{justify-content:space-around !important}.justify-content-xl-evenly{justify-content:space-evenly !important}.align-items-xl-start{align-items:flex-start !important}.align-items-xl-end{align-items:flex-end !important}.align-items-xl-center{align-items:center !important}.align-items-xl-baseline{align-items:baseline !important}.align-items-xl-stretch{align-items:stretch !important}.align-content-xl-start{align-content:flex-start !important}.align-content-xl-end{align-content:flex-end !important}.align-content-xl-center{align-content:center !important}.align-content-xl-between{align-content:space-between !important}.align-content-xl-around{align-content:space-around !important}.align-content-xl-stretch{align-content:stretch !important}.align-self-xl-auto{align-self:auto !important}.align-self-xl-start{align-self:flex-start !important}.align-self-xl-end{align-self:flex-end !important}.align-self-xl-center{align-self:center !important}.align-self-xl-baseline{align-self:baseline !important}.align-self-xl-stretch{align-self:stretch !important}.order-xl-first{order:-1 !important}.order-xl-0{order:0 !important}.order-xl-1{order:1 !important}.order-xl-2{order:2 !important}.order-xl-3{order:3 !important}.order-xl-4{order:4 !important}.order-xl-5{order:5 !important}.order-xl-last{order:6 !important}.m-xl-0{margin:0 !important}.m-xl-1{margin:.25rem !important}.m-xl-2{margin:.5rem !important}.m-xl-3{margin:1rem !important}.m-xl-4{margin:1.5rem !important}.m-xl-5{margin:3rem !important}.m-xl-auto{margin:auto !important}.mx-xl-0{margin-right:0 !important;margin-left:0 !important}.mx-xl-1{margin-right:.25rem !important;margin-left:.25rem !important}.mx-xl-2{margin-right:.5rem !important;margin-left:.5rem !important}.mx-xl-3{margin-right:1rem !important;margin-left:1rem !important}.mx-xl-4{margin-right:1.5rem !important;margin-left:1.5rem !important}.mx-xl-5{margin-right:3rem !important;margin-left:3rem !important}.mx-xl-auto{margin-right:auto !important;margin-left:auto !important}.my-xl-0{margin-top:0 !important;margin-bottom:0 !important}.my-xl-1{margin-top:.25rem !important;margin-bottom:.25rem !important}.my-xl-2{margin-top:.5rem !important;margin-bottom:.5rem !important}.my-xl-3{margin-top:1rem !important;margin-bottom:1rem !important}.my-xl-4{margin-top:1.5rem !important;margin-bottom:1.5rem !important}.my-xl-5{margin-top:3rem !important;margin-bottom:3rem !important}.my-xl-auto{margin-top:auto !important;margin-bottom:auto !important}.mt-xl-0{margin-top:0 !important}.mt-xl-1{margin-top:.25rem !important}.mt-xl-2{margin-top:.5rem !important}.mt-xl-3{margin-top:1rem !important}.mt-xl-4{margin-top:1.5rem !important}.mt-xl-5{margin-top:3rem !important}.mt-xl-auto{margin-top:auto !important}.me-xl-0{margin-right:0 !important}.me-xl-1{margin-right:.25rem !important}.me-xl-2{margin-right:.5rem !important}.me-xl-3{margin-right:1rem !important}.me-xl-4{margin-right:1.5rem !important}.me-xl-5{margin-right:3rem !important}.me-xl-auto{margin-right:auto !important}.mb-xl-0{margin-bottom:0 !important}.mb-xl-1{margin-bottom:.25rem !important}.mb-xl-2{margin-bottom:.5rem !important}.mb-xl-3{margin-bottom:1rem !important}.mb-xl-4{margin-bottom:1.5rem !important}.mb-xl-5{margin-bottom:3rem !important}.mb-xl-auto{margin-bottom:auto !important}.ms-xl-0{margin-left:0 !important}.ms-xl-1{margin-left:.25rem !important}.ms-xl-2{margin-left:.5rem !important}.ms-xl-3{margin-left:1rem !important}.ms-xl-4{margin-left:1.5rem !important}.ms-xl-5{margin-left:3rem !important}.ms-xl-auto{margin-left:auto !important}.p-xl-0{padding:0 !important}.p-xl-1{padding:.25rem !important}.p-xl-2{padding:.5rem !important}.p-xl-3{padding:1rem !important}.p-xl-4{padding:1.5rem !important}.p-xl-5{padding:3rem !important}.px-xl-0{padding-right:0 !important;padding-left:0 !important}.px-xl-1{padding-right:.25rem !important;padding-left:.25rem !important}.px-xl-2{padding-right:.5rem !important;padding-left:.5rem !important}.px-xl-3{padding-right:1rem !important;padding-left:1rem !important}.px-xl-4{padding-right:1.5rem !important;padding-left:1.5rem !important}.px-xl-5{padding-right:3rem !important;padding-left:3rem !important}.py-xl-0{padding-top:0 !important;padding-bottom:0 !important}.py-xl-1{padding-top:.25rem !important;padding-bottom:.25rem !important}.py-xl-2{padding-top:.5rem !important;padding-bottom:.5rem !important}.py-xl-3{padding-top:1rem !important;padding-bottom:1rem !important}.py-xl-4{padding-top:1.5rem !important;padding-bottom:1.5rem !important}.py-xl-5{padding-top:3rem !important;padding-bottom:3rem !important}.pt-xl-0{padding-top:0 !important}.pt-xl-1{padding-top:.25rem !important}.pt-xl-2{padding-top:.5rem !important}.pt-xl-3{padding-top:1rem !important}.pt-xl-4{padding-top:1.5rem !important}.pt-xl-5{padding-top:3rem !important}.pe-xl-0{padding-right:0 !important}.pe-xl-1{padding-right:.25rem !important}.pe-xl-2{padding-right:.5rem !important}.pe-xl-3{padding-right:1rem !important}.pe-xl-4{padding-right:1.5rem !important}.pe-xl-5{padding-right:3rem !important}.pb-xl-0{padding-bottom:0 !important}.pb-xl-1{padding-bottom:.25rem !important}.pb-xl-2{padding-bottom:.5rem !important}.pb-xl-3{padding-bottom:1rem !important}.pb-xl-4{padding-bottom:1.5rem !important}.pb-xl-5{padding-bottom:3rem !important}.ps-xl-0{padding-left:0 !important}.ps-xl-1{padding-left:.25rem !important}.ps-xl-2{padding-left:.5rem !important}.ps-xl-3{padding-left:1rem !important}.ps-xl-4{padding-left:1.5rem !important}.ps-xl-5{padding-left:3rem !important}.gap-xl-0{gap:0 !important}.gap-xl-1{gap:.25rem !important}.gap-xl-2{gap:.5rem !important}.gap-xl-3{gap:1rem !important}.gap-xl-4{gap:1.5rem !important}.gap-xl-5{gap:3rem !important}.row-gap-xl-0{row-gap:0 !important}.row-gap-xl-1{row-gap:.25rem !important}.row-gap-xl-2{row-gap:.5rem !important}.row-gap-xl-3{row-gap:1rem !important}.row-gap-xl-4{row-gap:1.5rem !important}.row-gap-xl-5{row-gap:3rem !important}.column-gap-xl-0{column-gap:0 !important}.column-gap-xl-1{column-gap:.25rem !important}.column-gap-xl-2{column-gap:.5rem !important}.column-gap-xl-3{column-gap:1rem !important}.column-gap-xl-4{column-gap:1.5rem !important}.column-gap-xl-5{column-gap:3rem !important}.text-xl-start{text-align:left !important}.text-xl-end{text-align:right !important}.text-xl-center{text-align:center !important}}@media(min-width: 1400px){.float-xxl-start{float:left !important}.float-xxl-end{float:right !important}.float-xxl-none{float:none !important}.object-fit-xxl-contain{object-fit:contain !important}.object-fit-xxl-cover{object-fit:cover !important}.object-fit-xxl-fill{object-fit:fill !important}.object-fit-xxl-scale{object-fit:scale-down !important}.object-fit-xxl-none{object-fit:none !important}.d-xxl-inline{display:inline !important}.d-xxl-inline-block{display:inline-block !important}.d-xxl-block{display:block !important}.d-xxl-grid{display:grid !important}.d-xxl-inline-grid{display:inline-grid !important}.d-xxl-table{display:table !important}.d-xxl-table-row{display:table-row !important}.d-xxl-table-cell{display:table-cell !important}.d-xxl-flex{display:flex !important}.d-xxl-inline-flex{display:inline-flex !important}.d-xxl-none{display:none !important}.flex-xxl-fill{flex:1 1 auto !important}.flex-xxl-row{flex-direction:row !important}.flex-xxl-column{flex-direction:column !important}.flex-xxl-row-reverse{flex-direction:row-reverse !important}.flex-xxl-column-reverse{flex-direction:column-reverse !important}.flex-xxl-grow-0{flex-grow:0 !important}.flex-xxl-grow-1{flex-grow:1 !important}.flex-xxl-shrink-0{flex-shrink:0 !important}.flex-xxl-shrink-1{flex-shrink:1 !important}.flex-xxl-wrap{flex-wrap:wrap !important}.flex-xxl-nowrap{flex-wrap:nowrap !important}.flex-xxl-wrap-reverse{flex-wrap:wrap-reverse !important}.justify-content-xxl-start{justify-content:flex-start !important}.justify-content-xxl-end{justify-content:flex-end !important}.justify-content-xxl-center{justify-content:center !important}.justify-content-xxl-between{justify-content:space-between !important}.justify-content-xxl-around{justify-content:space-around !important}.justify-content-xxl-evenly{justify-content:space-evenly !important}.align-items-xxl-start{align-items:flex-start !important}.align-items-xxl-end{align-items:flex-end !important}.align-items-xxl-center{align-items:center !important}.align-items-xxl-baseline{align-items:baseline !important}.align-items-xxl-stretch{align-items:stretch !important}.align-content-xxl-start{align-content:flex-start !important}.align-content-xxl-end{align-content:flex-end !important}.align-content-xxl-center{align-content:center !important}.align-content-xxl-between{align-content:space-between !important}.align-content-xxl-around{align-content:space-around !important}.align-content-xxl-stretch{align-content:stretch !important}.align-self-xxl-auto{align-self:auto !important}.align-self-xxl-start{align-self:flex-start !important}.align-self-xxl-end{align-self:flex-end !important}.align-self-xxl-center{align-self:center !important}.align-self-xxl-baseline{align-self:baseline !important}.align-self-xxl-stretch{align-self:stretch !important}.order-xxl-first{order:-1 !important}.order-xxl-0{order:0 !important}.order-xxl-1{order:1 !important}.order-xxl-2{order:2 !important}.order-xxl-3{order:3 !important}.order-xxl-4{order:4 !important}.order-xxl-5{order:5 !important}.order-xxl-last{order:6 !important}.m-xxl-0{margin:0 !important}.m-xxl-1{margin:.25rem !important}.m-xxl-2{margin:.5rem !important}.m-xxl-3{margin:1rem !important}.m-xxl-4{margin:1.5rem !important}.m-xxl-5{margin:3rem !important}.m-xxl-auto{margin:auto !important}.mx-xxl-0{margin-right:0 !important;margin-left:0 !important}.mx-xxl-1{margin-right:.25rem !important;margin-left:.25rem !important}.mx-xxl-2{margin-right:.5rem !important;margin-left:.5rem !important}.mx-xxl-3{margin-right:1rem !important;margin-left:1rem !important}.mx-xxl-4{margin-right:1.5rem !important;margin-left:1.5rem !important}.mx-xxl-5{margin-right:3rem !important;margin-left:3rem !important}.mx-xxl-auto{margin-right:auto !important;margin-left:auto !important}.my-xxl-0{margin-top:0 !important;margin-bottom:0 !important}.my-xxl-1{margin-top:.25rem !important;margin-bottom:.25rem !important}.my-xxl-2{margin-top:.5rem !important;margin-bottom:.5rem !important}.my-xxl-3{margin-top:1rem !important;margin-bottom:1rem !important}.my-xxl-4{margin-top:1.5rem !important;margin-bottom:1.5rem !important}.my-xxl-5{margin-top:3rem !important;margin-bottom:3rem !important}.my-xxl-auto{margin-top:auto !important;margin-bottom:auto !important}.mt-xxl-0{margin-top:0 !important}.mt-xxl-1{margin-top:.25rem !important}.mt-xxl-2{margin-top:.5rem !important}.mt-xxl-3{margin-top:1rem !important}.mt-xxl-4{margin-top:1.5rem !important}.mt-xxl-5{margin-top:3rem !important}.mt-xxl-auto{margin-top:auto !important}.me-xxl-0{margin-right:0 !important}.me-xxl-1{margin-right:.25rem !important}.me-xxl-2{margin-right:.5rem !important}.me-xxl-3{margin-right:1rem !important}.me-xxl-4{margin-right:1.5rem !important}.me-xxl-5{margin-right:3rem !important}.me-xxl-auto{margin-right:auto !important}.mb-xxl-0{margin-bottom:0 !important}.mb-xxl-1{margin-bottom:.25rem !important}.mb-xxl-2{margin-bottom:.5rem !important}.mb-xxl-3{margin-bottom:1rem !important}.mb-xxl-4{margin-bottom:1.5rem !important}.mb-xxl-5{margin-bottom:3rem !important}.mb-xxl-auto{margin-bottom:auto !important}.ms-xxl-0{margin-left:0 !important}.ms-xxl-1{margin-left:.25rem !important}.ms-xxl-2{margin-left:.5rem !important}.ms-xxl-3{margin-left:1rem !important}.ms-xxl-4{margin-left:1.5rem !important}.ms-xxl-5{margin-left:3rem !important}.ms-xxl-auto{margin-left:auto !important}.p-xxl-0{padding:0 !important}.p-xxl-1{padding:.25rem !important}.p-xxl-2{padding:.5rem !important}.p-xxl-3{padding:1rem !important}.p-xxl-4{padding:1.5rem !important}.p-xxl-5{padding:3rem !important}.px-xxl-0{padding-right:0 !important;padding-left:0 !important}.px-xxl-1{padding-right:.25rem !important;padding-left:.25rem !important}.px-xxl-2{padding-right:.5rem !important;padding-left:.5rem !important}.px-xxl-3{padding-right:1rem !important;padding-left:1rem !important}.px-xxl-4{padding-right:1.5rem !important;padding-left:1.5rem !important}.px-xxl-5{padding-right:3rem !important;padding-left:3rem !important}.py-xxl-0{padding-top:0 !important;padding-bottom:0 !important}.py-xxl-1{padding-top:.25rem !important;padding-bottom:.25rem !important}.py-xxl-2{padding-top:.5rem !important;padding-bottom:.5rem !important}.py-xxl-3{padding-top:1rem !important;padding-bottom:1rem !important}.py-xxl-4{padding-top:1.5rem !important;padding-bottom:1.5rem !important}.py-xxl-5{padding-top:3rem !important;padding-bottom:3rem !important}.pt-xxl-0{padding-top:0 !important}.pt-xxl-1{padding-top:.25rem !important}.pt-xxl-2{padding-top:.5rem !important}.pt-xxl-3{padding-top:1rem !important}.pt-xxl-4{padding-top:1.5rem !important}.pt-xxl-5{padding-top:3rem !important}.pe-xxl-0{padding-right:0 !important}.pe-xxl-1{padding-right:.25rem !important}.pe-xxl-2{padding-right:.5rem !important}.pe-xxl-3{padding-right:1rem !important}.pe-xxl-4{padding-right:1.5rem !important}.pe-xxl-5{padding-right:3rem !important}.pb-xxl-0{padding-bottom:0 !important}.pb-xxl-1{padding-bottom:.25rem !important}.pb-xxl-2{padding-bottom:.5rem !important}.pb-xxl-3{padding-bottom:1rem !important}.pb-xxl-4{padding-bottom:1.5rem !important}.pb-xxl-5{padding-bottom:3rem !important}.ps-xxl-0{padding-left:0 !important}.ps-xxl-1{padding-left:.25rem !important}.ps-xxl-2{padding-left:.5rem !important}.ps-xxl-3{padding-left:1rem !important}.ps-xxl-4{padding-left:1.5rem !important}.ps-xxl-5{padding-left:3rem !important}.gap-xxl-0{gap:0 !important}.gap-xxl-1{gap:.25rem !important}.gap-xxl-2{gap:.5rem !important}.gap-xxl-3{gap:1rem !important}.gap-xxl-4{gap:1.5rem !important}.gap-xxl-5{gap:3rem !important}.row-gap-xxl-0{row-gap:0 !important}.row-gap-xxl-1{row-gap:.25rem !important}.row-gap-xxl-2{row-gap:.5rem !important}.row-gap-xxl-3{row-gap:1rem !important}.row-gap-xxl-4{row-gap:1.5rem !important}.row-gap-xxl-5{row-gap:3rem !important}.column-gap-xxl-0{column-gap:0 !important}.column-gap-xxl-1{column-gap:.25rem !important}.column-gap-xxl-2{column-gap:.5rem !important}.column-gap-xxl-3{column-gap:1rem !important}.column-gap-xxl-4{column-gap:1.5rem !important}.column-gap-xxl-5{column-gap:3rem !important}.text-xxl-start{text-align:left !important}.text-xxl-end{text-align:right !important}.text-xxl-center{text-align:center !important}}.bg-default{color:#fff}.bg-primary{color:#fff}.bg-secondary{color:#fff}.bg-success{color:#fff}.bg-info{color:#fff}.bg-warning{color:#fff}.bg-danger{color:#fff}.bg-light{color:#000}.bg-dark{color:#fff}@media(min-width: 1200px){.fs-1{font-size:2rem !important}.fs-2{font-size:1.65rem !important}.fs-3{font-size:1.45rem !important}}@media print{.d-print-inline{display:inline !important}.d-print-inline-block{display:inline-block !important}.d-print-block{display:block !important}.d-print-grid{display:grid !important}.d-print-inline-grid{display:inline-grid !important}.d-print-table{display:table !important}.d-print-table-row{display:table-row !important}.d-print-table-cell{display:table-cell !important}.d-print-flex{display:flex !important}.d-print-inline-flex{display:inline-flex !important}.d-print-none{display:none !important}}:root{--bslib-spacer: 1rem;--bslib-mb-spacer: var(--bslib-spacer, 1rem)}.bslib-mb-spacing{margin-bottom:var(--bslib-mb-spacer)}.bslib-gap-spacing{gap:var(--bslib-mb-spacer)}.bslib-gap-spacing>.bslib-mb-spacing,.bslib-gap-spacing>.form-group,.bslib-gap-spacing>p,.bslib-gap-spacing>pre{margin-bottom:0}.html-fill-container>.html-fill-item.bslib-mb-spacing{margin-bottom:0}.tab-content>.tab-pane.html-fill-container{display:none}.tab-content>.active.html-fill-container{display:flex}.tab-content.html-fill-container{padding:0}.bg-blue{--bslib-color-bg: #2780e3;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-blue{--bslib-color-fg: #2780e3;color:var(--bslib-color-fg)}.bg-indigo{--bslib-color-bg: #6610f2;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-indigo{--bslib-color-fg: #6610f2;color:var(--bslib-color-fg)}.bg-purple{--bslib-color-bg: #613d7c;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-purple{--bslib-color-fg: #613d7c;color:var(--bslib-color-fg)}.bg-pink{--bslib-color-bg: #e83e8c;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-pink{--bslib-color-fg: #e83e8c;color:var(--bslib-color-fg)}.bg-red{--bslib-color-bg: #ff0039;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-red{--bslib-color-fg: #ff0039;color:var(--bslib-color-fg)}.bg-orange{--bslib-color-bg: #f0ad4e;--bslib-color-fg: #000;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-orange{--bslib-color-fg: #f0ad4e;color:var(--bslib-color-fg)}.bg-yellow{--bslib-color-bg: #ff7518;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-yellow{--bslib-color-fg: #ff7518;color:var(--bslib-color-fg)}.bg-green{--bslib-color-bg: #3fb618;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-green{--bslib-color-fg: #3fb618;color:var(--bslib-color-fg)}.bg-teal{--bslib-color-bg: #20c997;--bslib-color-fg: #000;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-teal{--bslib-color-fg: #20c997;color:var(--bslib-color-fg)}.bg-cyan{--bslib-color-bg: #9954bb;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-cyan{--bslib-color-fg: #9954bb;color:var(--bslib-color-fg)}.text-default{--bslib-color-fg: #343a40}.bg-default{--bslib-color-bg: #343a40;--bslib-color-fg: #fff}.text-primary{--bslib-color-fg: #2780e3}.bg-primary{--bslib-color-bg: #2780e3;--bslib-color-fg: #fff}.text-secondary{--bslib-color-fg: #343a40}.bg-secondary{--bslib-color-bg: #343a40;--bslib-color-fg: #fff}.text-success{--bslib-color-fg: #3fb618}.bg-success{--bslib-color-bg: #3fb618;--bslib-color-fg: #fff}.text-info{--bslib-color-fg: #9954bb}.bg-info{--bslib-color-bg: #9954bb;--bslib-color-fg: #fff}.text-warning{--bslib-color-fg: #ff7518}.bg-warning{--bslib-color-bg: #ff7518;--bslib-color-fg: #fff}.text-danger{--bslib-color-fg: #ff0039}.bg-danger{--bslib-color-bg: #ff0039;--bslib-color-fg: #fff}.text-light{--bslib-color-fg: #f8f9fa}.bg-light{--bslib-color-bg: #f8f9fa;--bslib-color-fg: #000}.text-dark{--bslib-color-fg: #343a40}.bg-dark{--bslib-color-bg: #343a40;--bslib-color-fg: #fff}.bg-gradient-blue-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #4053e9;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #4053e9;color:#fff}.bg-gradient-blue-purple{--bslib-color-fg: #fff;--bslib-color-bg: #3e65ba;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #3e65ba;color:#fff}.bg-gradient-blue-pink{--bslib-color-fg: #fff;--bslib-color-bg: #7466c0;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #7466c0;color:#fff}.bg-gradient-blue-red{--bslib-color-fg: #fff;--bslib-color-bg: #7d4d9f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #7d4d9f;color:#fff}.bg-gradient-blue-orange{--bslib-color-fg: #fff;--bslib-color-bg: #7792a7;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #7792a7;color:#fff}.bg-gradient-blue-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #7d7c92;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #7d7c92;color:#fff}.bg-gradient-blue-green{--bslib-color-fg: #fff;--bslib-color-bg: #319692;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #319692;color:#fff}.bg-gradient-blue-teal{--bslib-color-fg: #fff;--bslib-color-bg: #249dc5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #249dc5;color:#fff}.bg-gradient-blue-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #556ed3;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #556ed3;color:#fff}.bg-gradient-indigo-blue{--bslib-color-fg: #fff;--bslib-color-bg: #4d3dec;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #4d3dec;color:#fff}.bg-gradient-indigo-purple{--bslib-color-fg: #fff;--bslib-color-bg: #6422c3;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #6422c3;color:#fff}.bg-gradient-indigo-pink{--bslib-color-fg: #fff;--bslib-color-bg: #9a22c9;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #9a22c9;color:#fff}.bg-gradient-indigo-red{--bslib-color-fg: #fff;--bslib-color-bg: #a30aa8;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #a30aa8;color:#fff}.bg-gradient-indigo-orange{--bslib-color-fg: #fff;--bslib-color-bg: #9d4fb0;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #9d4fb0;color:#fff}.bg-gradient-indigo-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #a3389b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #a3389b;color:#fff}.bg-gradient-indigo-green{--bslib-color-fg: #fff;--bslib-color-bg: #56529b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #56529b;color:#fff}.bg-gradient-indigo-teal{--bslib-color-fg: #fff;--bslib-color-bg: #4a5ace;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #4a5ace;color:#fff}.bg-gradient-indigo-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #7a2bdc;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #7a2bdc;color:#fff}.bg-gradient-purple-blue{--bslib-color-fg: #fff;--bslib-color-bg: #4a58a5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #4a58a5;color:#fff}.bg-gradient-purple-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #632bab;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #632bab;color:#fff}.bg-gradient-purple-pink{--bslib-color-fg: #fff;--bslib-color-bg: #973d82;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #973d82;color:#fff}.bg-gradient-purple-red{--bslib-color-fg: #fff;--bslib-color-bg: #a02561;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #a02561;color:#fff}.bg-gradient-purple-orange{--bslib-color-fg: #fff;--bslib-color-bg: #9a6a6a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #9a6a6a;color:#fff}.bg-gradient-purple-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #a05354;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #a05354;color:#fff}.bg-gradient-purple-green{--bslib-color-fg: #fff;--bslib-color-bg: #536d54;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #536d54;color:#fff}.bg-gradient-purple-teal{--bslib-color-fg: #fff;--bslib-color-bg: #477587;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #477587;color:#fff}.bg-gradient-purple-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #774695;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #774695;color:#fff}.bg-gradient-pink-blue{--bslib-color-fg: #fff;--bslib-color-bg: #9b58af;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #9b58af;color:#fff}.bg-gradient-pink-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #b42cb5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #b42cb5;color:#fff}.bg-gradient-pink-purple{--bslib-color-fg: #fff;--bslib-color-bg: #b23e86;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #b23e86;color:#fff}.bg-gradient-pink-red{--bslib-color-fg: #fff;--bslib-color-bg: #f1256b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #f1256b;color:#fff}.bg-gradient-pink-orange{--bslib-color-fg: #fff;--bslib-color-bg: #eb6a73;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #eb6a73;color:#fff}.bg-gradient-pink-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #f1545e;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #f1545e;color:#fff}.bg-gradient-pink-green{--bslib-color-fg: #fff;--bslib-color-bg: #a46e5e;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #a46e5e;color:#fff}.bg-gradient-pink-teal{--bslib-color-fg: #fff;--bslib-color-bg: #987690;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #987690;color:#fff}.bg-gradient-pink-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #c8479f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #c8479f;color:#fff}.bg-gradient-red-blue{--bslib-color-fg: #fff;--bslib-color-bg: #a9337d;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #a9337d;color:#fff}.bg-gradient-red-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #c20683;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #c20683;color:#fff}.bg-gradient-red-purple{--bslib-color-fg: #fff;--bslib-color-bg: #c01854;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #c01854;color:#fff}.bg-gradient-red-pink{--bslib-color-fg: #fff;--bslib-color-bg: #f6195a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #f6195a;color:#fff}.bg-gradient-red-orange{--bslib-color-fg: #fff;--bslib-color-bg: #f94541;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #f94541;color:#fff}.bg-gradient-red-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #ff2f2c;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #ff2f2c;color:#fff}.bg-gradient-red-green{--bslib-color-fg: #fff;--bslib-color-bg: #b2492c;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #b2492c;color:#fff}.bg-gradient-red-teal{--bslib-color-fg: #fff;--bslib-color-bg: #a6505f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #a6505f;color:#fff}.bg-gradient-red-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #d6226d;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #d6226d;color:#fff}.bg-gradient-orange-blue{--bslib-color-fg: #fff;--bslib-color-bg: #a09b8a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #a09b8a;color:#fff}.bg-gradient-orange-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #b96e90;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #b96e90;color:#fff}.bg-gradient-orange-purple{--bslib-color-fg: #fff;--bslib-color-bg: #b78060;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #b78060;color:#fff}.bg-gradient-orange-pink{--bslib-color-fg: #fff;--bslib-color-bg: #ed8167;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #ed8167;color:#fff}.bg-gradient-orange-red{--bslib-color-fg: #fff;--bslib-color-bg: #f66846;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #f66846;color:#fff}.bg-gradient-orange-yellow{--bslib-color-fg: #000;--bslib-color-bg: #f69738;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #f69738;color:#000}.bg-gradient-orange-green{--bslib-color-fg: #000;--bslib-color-bg: #a9b138;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #a9b138;color:#000}.bg-gradient-orange-teal{--bslib-color-fg: #000;--bslib-color-bg: #9db86b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #9db86b;color:#000}.bg-gradient-orange-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #cd897a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #cd897a;color:#fff}.bg-gradient-yellow-blue{--bslib-color-fg: #fff;--bslib-color-bg: #a97969;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #a97969;color:#fff}.bg-gradient-yellow-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #c24d6f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #c24d6f;color:#fff}.bg-gradient-yellow-purple{--bslib-color-fg: #fff;--bslib-color-bg: #c05f40;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #c05f40;color:#fff}.bg-gradient-yellow-pink{--bslib-color-fg: #fff;--bslib-color-bg: #f65f46;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #f65f46;color:#fff}.bg-gradient-yellow-red{--bslib-color-fg: #fff;--bslib-color-bg: #ff4625;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #ff4625;color:#fff}.bg-gradient-yellow-orange{--bslib-color-fg: #000;--bslib-color-bg: #f98b2e;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #f98b2e;color:#000}.bg-gradient-yellow-green{--bslib-color-fg: #fff;--bslib-color-bg: #b28f18;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #b28f18;color:#fff}.bg-gradient-yellow-teal{--bslib-color-fg: #fff;--bslib-color-bg: #a6974b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #a6974b;color:#fff}.bg-gradient-yellow-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #d66859;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #d66859;color:#fff}.bg-gradient-green-blue{--bslib-color-fg: #fff;--bslib-color-bg: #35a069;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #35a069;color:#fff}.bg-gradient-green-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #4f746f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #4f746f;color:#fff}.bg-gradient-green-purple{--bslib-color-fg: #fff;--bslib-color-bg: #4d8640;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #4d8640;color:#fff}.bg-gradient-green-pink{--bslib-color-fg: #fff;--bslib-color-bg: #838646;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #838646;color:#fff}.bg-gradient-green-red{--bslib-color-fg: #fff;--bslib-color-bg: #8c6d25;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #8c6d25;color:#fff}.bg-gradient-green-orange{--bslib-color-fg: #000;--bslib-color-bg: #86b22e;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #86b22e;color:#000}.bg-gradient-green-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #8c9c18;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #8c9c18;color:#fff}.bg-gradient-green-teal{--bslib-color-fg: #000;--bslib-color-bg: #33be4b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #33be4b;color:#000}.bg-gradient-green-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #638f59;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #638f59;color:#fff}.bg-gradient-teal-blue{--bslib-color-fg: #fff;--bslib-color-bg: #23acb5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #23acb5;color:#fff}.bg-gradient-teal-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #3c7fbb;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #3c7fbb;color:#fff}.bg-gradient-teal-purple{--bslib-color-fg: #fff;--bslib-color-bg: #3a918c;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #3a918c;color:#fff}.bg-gradient-teal-pink{--bslib-color-fg: #fff;--bslib-color-bg: #709193;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #709193;color:#fff}.bg-gradient-teal-red{--bslib-color-fg: #fff;--bslib-color-bg: #797971;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #797971;color:#fff}.bg-gradient-teal-orange{--bslib-color-fg: #000;--bslib-color-bg: #73be7a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #73be7a;color:#000}.bg-gradient-teal-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #79a764;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #79a764;color:#fff}.bg-gradient-teal-green{--bslib-color-fg: #000;--bslib-color-bg: #2cc164;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #2cc164;color:#000}.bg-gradient-teal-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #509aa5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #509aa5;color:#fff}.bg-gradient-cyan-blue{--bslib-color-fg: #fff;--bslib-color-bg: #6b66cb;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #6b66cb;color:#fff}.bg-gradient-cyan-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #8539d1;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #8539d1;color:#fff}.bg-gradient-cyan-purple{--bslib-color-fg: #fff;--bslib-color-bg: #834ba2;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #834ba2;color:#fff}.bg-gradient-cyan-pink{--bslib-color-fg: #fff;--bslib-color-bg: #b94ba8;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #b94ba8;color:#fff}.bg-gradient-cyan-red{--bslib-color-fg: #fff;--bslib-color-bg: #c23287;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #c23287;color:#fff}.bg-gradient-cyan-orange{--bslib-color-fg: #fff;--bslib-color-bg: #bc788f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #bc788f;color:#fff}.bg-gradient-cyan-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #c2617a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #c2617a;color:#fff}.bg-gradient-cyan-green{--bslib-color-fg: #fff;--bslib-color-bg: #757b7a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #757b7a;color:#fff}.bg-gradient-cyan-teal{--bslib-color-fg: #fff;--bslib-color-bg: #6983ad;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #6983ad;color:#fff}.tab-content>.tab-pane.html-fill-container{display:none}.tab-content>.active.html-fill-container{display:flex}.tab-content.html-fill-container{padding:0}:root{--bslib-spacer: 1rem;--bslib-mb-spacer: var(--bslib-spacer, 1rem)}.bslib-mb-spacing{margin-bottom:var(--bslib-mb-spacer)}.bslib-gap-spacing{gap:var(--bslib-mb-spacer)}.bslib-gap-spacing>.bslib-mb-spacing,.bslib-gap-spacing>.form-group,.bslib-gap-spacing>p,.bslib-gap-spacing>pre{margin-bottom:0}.html-fill-container>.html-fill-item.bslib-mb-spacing{margin-bottom:0}.bg-blue{--bslib-color-bg: #2780e3;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-blue{--bslib-color-fg: #2780e3;color:var(--bslib-color-fg)}.bg-indigo{--bslib-color-bg: #6610f2;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-indigo{--bslib-color-fg: #6610f2;color:var(--bslib-color-fg)}.bg-purple{--bslib-color-bg: #613d7c;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-purple{--bslib-color-fg: #613d7c;color:var(--bslib-color-fg)}.bg-pink{--bslib-color-bg: #e83e8c;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-pink{--bslib-color-fg: #e83e8c;color:var(--bslib-color-fg)}.bg-red{--bslib-color-bg: #ff0039;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-red{--bslib-color-fg: #ff0039;color:var(--bslib-color-fg)}.bg-orange{--bslib-color-bg: #f0ad4e;--bslib-color-fg: #000;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-orange{--bslib-color-fg: #f0ad4e;color:var(--bslib-color-fg)}.bg-yellow{--bslib-color-bg: #ff7518;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-yellow{--bslib-color-fg: #ff7518;color:var(--bslib-color-fg)}.bg-green{--bslib-color-bg: #3fb618;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-green{--bslib-color-fg: #3fb618;color:var(--bslib-color-fg)}.bg-teal{--bslib-color-bg: #20c997;--bslib-color-fg: #000;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-teal{--bslib-color-fg: #20c997;color:var(--bslib-color-fg)}.bg-cyan{--bslib-color-bg: #9954bb;--bslib-color-fg: #fff;background-color:var(--bslib-color-bg);color:var(--bslib-color-fg)}.text-cyan{--bslib-color-fg: #9954bb;color:var(--bslib-color-fg)}.text-default{--bslib-color-fg: #343a40}.bg-default{--bslib-color-bg: #343a40;--bslib-color-fg: #fff}.text-primary{--bslib-color-fg: #2780e3}.bg-primary{--bslib-color-bg: #2780e3;--bslib-color-fg: #fff}.text-secondary{--bslib-color-fg: #343a40}.bg-secondary{--bslib-color-bg: #343a40;--bslib-color-fg: #fff}.text-success{--bslib-color-fg: #3fb618}.bg-success{--bslib-color-bg: #3fb618;--bslib-color-fg: #fff}.text-info{--bslib-color-fg: #9954bb}.bg-info{--bslib-color-bg: #9954bb;--bslib-color-fg: #fff}.text-warning{--bslib-color-fg: #ff7518}.bg-warning{--bslib-color-bg: #ff7518;--bslib-color-fg: #fff}.text-danger{--bslib-color-fg: #ff0039}.bg-danger{--bslib-color-bg: #ff0039;--bslib-color-fg: #fff}.text-light{--bslib-color-fg: #f8f9fa}.bg-light{--bslib-color-bg: #f8f9fa;--bslib-color-fg: #000}.text-dark{--bslib-color-fg: #343a40}.bg-dark{--bslib-color-bg: #343a40;--bslib-color-fg: #fff}.bg-gradient-blue-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #4053e9;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #4053e9;color:#fff}.bg-gradient-blue-purple{--bslib-color-fg: #fff;--bslib-color-bg: #3e65ba;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #3e65ba;color:#fff}.bg-gradient-blue-pink{--bslib-color-fg: #fff;--bslib-color-bg: #7466c0;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #7466c0;color:#fff}.bg-gradient-blue-red{--bslib-color-fg: #fff;--bslib-color-bg: #7d4d9f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #7d4d9f;color:#fff}.bg-gradient-blue-orange{--bslib-color-fg: #fff;--bslib-color-bg: #7792a7;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #7792a7;color:#fff}.bg-gradient-blue-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #7d7c92;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #7d7c92;color:#fff}.bg-gradient-blue-green{--bslib-color-fg: #fff;--bslib-color-bg: #319692;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #319692;color:#fff}.bg-gradient-blue-teal{--bslib-color-fg: #fff;--bslib-color-bg: #249dc5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #249dc5;color:#fff}.bg-gradient-blue-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #556ed3;background:linear-gradient(var(--bg-gradient-deg, 140deg), #2780e3 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #556ed3;color:#fff}.bg-gradient-indigo-blue{--bslib-color-fg: #fff;--bslib-color-bg: #4d3dec;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #4d3dec;color:#fff}.bg-gradient-indigo-purple{--bslib-color-fg: #fff;--bslib-color-bg: #6422c3;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #6422c3;color:#fff}.bg-gradient-indigo-pink{--bslib-color-fg: #fff;--bslib-color-bg: #9a22c9;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #9a22c9;color:#fff}.bg-gradient-indigo-red{--bslib-color-fg: #fff;--bslib-color-bg: #a30aa8;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #a30aa8;color:#fff}.bg-gradient-indigo-orange{--bslib-color-fg: #fff;--bslib-color-bg: #9d4fb0;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #9d4fb0;color:#fff}.bg-gradient-indigo-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #a3389b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #a3389b;color:#fff}.bg-gradient-indigo-green{--bslib-color-fg: #fff;--bslib-color-bg: #56529b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #56529b;color:#fff}.bg-gradient-indigo-teal{--bslib-color-fg: #fff;--bslib-color-bg: #4a5ace;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #4a5ace;color:#fff}.bg-gradient-indigo-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #7a2bdc;background:linear-gradient(var(--bg-gradient-deg, 140deg), #6610f2 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #7a2bdc;color:#fff}.bg-gradient-purple-blue{--bslib-color-fg: #fff;--bslib-color-bg: #4a58a5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #4a58a5;color:#fff}.bg-gradient-purple-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #632bab;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #632bab;color:#fff}.bg-gradient-purple-pink{--bslib-color-fg: #fff;--bslib-color-bg: #973d82;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #973d82;color:#fff}.bg-gradient-purple-red{--bslib-color-fg: #fff;--bslib-color-bg: #a02561;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #a02561;color:#fff}.bg-gradient-purple-orange{--bslib-color-fg: #fff;--bslib-color-bg: #9a6a6a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #9a6a6a;color:#fff}.bg-gradient-purple-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #a05354;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #a05354;color:#fff}.bg-gradient-purple-green{--bslib-color-fg: #fff;--bslib-color-bg: #536d54;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #536d54;color:#fff}.bg-gradient-purple-teal{--bslib-color-fg: #fff;--bslib-color-bg: #477587;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #477587;color:#fff}.bg-gradient-purple-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #774695;background:linear-gradient(var(--bg-gradient-deg, 140deg), #613d7c var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #774695;color:#fff}.bg-gradient-pink-blue{--bslib-color-fg: #fff;--bslib-color-bg: #9b58af;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #9b58af;color:#fff}.bg-gradient-pink-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #b42cb5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #b42cb5;color:#fff}.bg-gradient-pink-purple{--bslib-color-fg: #fff;--bslib-color-bg: #b23e86;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #b23e86;color:#fff}.bg-gradient-pink-red{--bslib-color-fg: #fff;--bslib-color-bg: #f1256b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #f1256b;color:#fff}.bg-gradient-pink-orange{--bslib-color-fg: #fff;--bslib-color-bg: #eb6a73;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #eb6a73;color:#fff}.bg-gradient-pink-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #f1545e;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #f1545e;color:#fff}.bg-gradient-pink-green{--bslib-color-fg: #fff;--bslib-color-bg: #a46e5e;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #a46e5e;color:#fff}.bg-gradient-pink-teal{--bslib-color-fg: #fff;--bslib-color-bg: #987690;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #987690;color:#fff}.bg-gradient-pink-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #c8479f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #e83e8c var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #c8479f;color:#fff}.bg-gradient-red-blue{--bslib-color-fg: #fff;--bslib-color-bg: #a9337d;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #a9337d;color:#fff}.bg-gradient-red-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #c20683;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #c20683;color:#fff}.bg-gradient-red-purple{--bslib-color-fg: #fff;--bslib-color-bg: #c01854;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #c01854;color:#fff}.bg-gradient-red-pink{--bslib-color-fg: #fff;--bslib-color-bg: #f6195a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #f6195a;color:#fff}.bg-gradient-red-orange{--bslib-color-fg: #fff;--bslib-color-bg: #f94541;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #f94541;color:#fff}.bg-gradient-red-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #ff2f2c;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #ff2f2c;color:#fff}.bg-gradient-red-green{--bslib-color-fg: #fff;--bslib-color-bg: #b2492c;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #b2492c;color:#fff}.bg-gradient-red-teal{--bslib-color-fg: #fff;--bslib-color-bg: #a6505f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #a6505f;color:#fff}.bg-gradient-red-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #d6226d;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff0039 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #d6226d;color:#fff}.bg-gradient-orange-blue{--bslib-color-fg: #fff;--bslib-color-bg: #a09b8a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #a09b8a;color:#fff}.bg-gradient-orange-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #b96e90;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #b96e90;color:#fff}.bg-gradient-orange-purple{--bslib-color-fg: #fff;--bslib-color-bg: #b78060;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #b78060;color:#fff}.bg-gradient-orange-pink{--bslib-color-fg: #fff;--bslib-color-bg: #ed8167;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #ed8167;color:#fff}.bg-gradient-orange-red{--bslib-color-fg: #fff;--bslib-color-bg: #f66846;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #f66846;color:#fff}.bg-gradient-orange-yellow{--bslib-color-fg: #000;--bslib-color-bg: #f69738;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #f69738;color:#000}.bg-gradient-orange-green{--bslib-color-fg: #000;--bslib-color-bg: #a9b138;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #a9b138;color:#000}.bg-gradient-orange-teal{--bslib-color-fg: #000;--bslib-color-bg: #9db86b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #9db86b;color:#000}.bg-gradient-orange-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #cd897a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #f0ad4e var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #cd897a;color:#fff}.bg-gradient-yellow-blue{--bslib-color-fg: #fff;--bslib-color-bg: #a97969;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #a97969;color:#fff}.bg-gradient-yellow-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #c24d6f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #c24d6f;color:#fff}.bg-gradient-yellow-purple{--bslib-color-fg: #fff;--bslib-color-bg: #c05f40;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #c05f40;color:#fff}.bg-gradient-yellow-pink{--bslib-color-fg: #fff;--bslib-color-bg: #f65f46;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #f65f46;color:#fff}.bg-gradient-yellow-red{--bslib-color-fg: #fff;--bslib-color-bg: #ff4625;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #ff4625;color:#fff}.bg-gradient-yellow-orange{--bslib-color-fg: #000;--bslib-color-bg: #f98b2e;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #f98b2e;color:#000}.bg-gradient-yellow-green{--bslib-color-fg: #fff;--bslib-color-bg: #b28f18;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #b28f18;color:#fff}.bg-gradient-yellow-teal{--bslib-color-fg: #fff;--bslib-color-bg: #a6974b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #a6974b;color:#fff}.bg-gradient-yellow-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #d66859;background:linear-gradient(var(--bg-gradient-deg, 140deg), #ff7518 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #d66859;color:#fff}.bg-gradient-green-blue{--bslib-color-fg: #fff;--bslib-color-bg: #35a069;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #35a069;color:#fff}.bg-gradient-green-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #4f746f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #4f746f;color:#fff}.bg-gradient-green-purple{--bslib-color-fg: #fff;--bslib-color-bg: #4d8640;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #4d8640;color:#fff}.bg-gradient-green-pink{--bslib-color-fg: #fff;--bslib-color-bg: #838646;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #838646;color:#fff}.bg-gradient-green-red{--bslib-color-fg: #fff;--bslib-color-bg: #8c6d25;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #8c6d25;color:#fff}.bg-gradient-green-orange{--bslib-color-fg: #000;--bslib-color-bg: #86b22e;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #86b22e;color:#000}.bg-gradient-green-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #8c9c18;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #8c9c18;color:#fff}.bg-gradient-green-teal{--bslib-color-fg: #000;--bslib-color-bg: #33be4b;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #33be4b;color:#000}.bg-gradient-green-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #638f59;background:linear-gradient(var(--bg-gradient-deg, 140deg), #3fb618 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #638f59;color:#fff}.bg-gradient-teal-blue{--bslib-color-fg: #fff;--bslib-color-bg: #23acb5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #23acb5;color:#fff}.bg-gradient-teal-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #3c7fbb;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #3c7fbb;color:#fff}.bg-gradient-teal-purple{--bslib-color-fg: #fff;--bslib-color-bg: #3a918c;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #3a918c;color:#fff}.bg-gradient-teal-pink{--bslib-color-fg: #fff;--bslib-color-bg: #709193;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #709193;color:#fff}.bg-gradient-teal-red{--bslib-color-fg: #fff;--bslib-color-bg: #797971;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #797971;color:#fff}.bg-gradient-teal-orange{--bslib-color-fg: #000;--bslib-color-bg: #73be7a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #73be7a;color:#000}.bg-gradient-teal-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #79a764;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #79a764;color:#fff}.bg-gradient-teal-green{--bslib-color-fg: #000;--bslib-color-bg: #2cc164;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #2cc164;color:#000}.bg-gradient-teal-cyan{--bslib-color-fg: #fff;--bslib-color-bg: #509aa5;background:linear-gradient(var(--bg-gradient-deg, 140deg), #20c997 var(--bg-gradient-start, 36%), #9954bb var(--bg-gradient-end, 180%)) #509aa5;color:#fff}.bg-gradient-cyan-blue{--bslib-color-fg: #fff;--bslib-color-bg: #6b66cb;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #2780e3 var(--bg-gradient-end, 180%)) #6b66cb;color:#fff}.bg-gradient-cyan-indigo{--bslib-color-fg: #fff;--bslib-color-bg: #8539d1;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #6610f2 var(--bg-gradient-end, 180%)) #8539d1;color:#fff}.bg-gradient-cyan-purple{--bslib-color-fg: #fff;--bslib-color-bg: #834ba2;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #613d7c var(--bg-gradient-end, 180%)) #834ba2;color:#fff}.bg-gradient-cyan-pink{--bslib-color-fg: #fff;--bslib-color-bg: #b94ba8;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #e83e8c var(--bg-gradient-end, 180%)) #b94ba8;color:#fff}.bg-gradient-cyan-red{--bslib-color-fg: #fff;--bslib-color-bg: #c23287;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #ff0039 var(--bg-gradient-end, 180%)) #c23287;color:#fff}.bg-gradient-cyan-orange{--bslib-color-fg: #fff;--bslib-color-bg: #bc788f;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #f0ad4e var(--bg-gradient-end, 180%)) #bc788f;color:#fff}.bg-gradient-cyan-yellow{--bslib-color-fg: #fff;--bslib-color-bg: #c2617a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #ff7518 var(--bg-gradient-end, 180%)) #c2617a;color:#fff}.bg-gradient-cyan-green{--bslib-color-fg: #fff;--bslib-color-bg: #757b7a;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #3fb618 var(--bg-gradient-end, 180%)) #757b7a;color:#fff}.bg-gradient-cyan-teal{--bslib-color-fg: #fff;--bslib-color-bg: #6983ad;background:linear-gradient(var(--bg-gradient-deg, 140deg), #9954bb var(--bg-gradient-start, 36%), #20c997 var(--bg-gradient-end, 180%)) #6983ad;color:#fff}:root{--bslib-value-box-shadow: none;--bslib-value-box-border-width-auto-yes: var(--bslib-value-box-border-width-baseline);--bslib-value-box-border-width-auto-no: 0;--bslib-value-box-border-width-baseline: 1px}.bslib-value-box{border-width:var(--bslib-value-box-border-width-auto-no, var(--bslib-value-box-border-width-baseline));container-name:bslib-value-box;container-type:inline-size}.bslib-value-box.card{box-shadow:var(--bslib-value-box-shadow)}.bslib-value-box.border-auto{border-width:var(--bslib-value-box-border-width-auto-yes, var(--bslib-value-box-border-width-baseline))}.bslib-value-box.default{--bslib-value-box-bg-default: var(--bs-card-bg, #fff);--bslib-value-box-border-color-default: var(--bs-card-border-color, rgba(0, 0, 0, 0.175));color:var(--bslib-value-box-color);background-color:var(--bslib-value-box-bg, var(--bslib-value-box-bg-default));border-color:var(--bslib-value-box-border-color, var(--bslib-value-box-border-color-default))}.bslib-value-box .value-box-grid{display:grid;grid-template-areas:"left right";align-items:center;overflow:hidden}.bslib-value-box .value-box-showcase{height:100%;max-height:var(---bslib-value-box-showcase-max-h, 100%)}.bslib-value-box .value-box-showcase,.bslib-value-box .value-box-showcase>.html-fill-item{width:100%}.bslib-value-box[data-full-screen=true] .value-box-showcase{max-height:var(---bslib-value-box-showcase-max-h-fs, 100%)}@media screen and (min-width: 575.98px){@container bslib-value-box (max-width: 300px){.bslib-value-box:not(.showcase-bottom) .value-box-grid{grid-template-columns:1fr !important;grid-template-rows:auto auto;grid-template-areas:"top" "bottom"}.bslib-value-box:not(.showcase-bottom) .value-box-grid .value-box-showcase{grid-area:top !important}.bslib-value-box:not(.showcase-bottom) .value-box-grid .value-box-area{grid-area:bottom !important;justify-content:end}}}.bslib-value-box .value-box-area{justify-content:center;padding:1.5rem 1rem;font-size:.9rem;font-weight:500}.bslib-value-box .value-box-area *{margin-bottom:0;margin-top:0}.bslib-value-box .value-box-title{font-size:1rem;margin-top:0;margin-bottom:.5rem;font-weight:400;line-height:1.2}.bslib-value-box .value-box-title:empty::after{content:" "}.bslib-value-box .value-box-value{font-size:calc(1.29rem + 0.48vw);margin-top:0;margin-bottom:.5rem;font-weight:400;line-height:1.2}@media(min-width: 1200px){.bslib-value-box .value-box-value{font-size:1.65rem}}.bslib-value-box .value-box-value:empty::after{content:" "}.bslib-value-box .value-box-showcase{align-items:center;justify-content:center;margin-top:auto;margin-bottom:auto;padding:1rem}.bslib-value-box .value-box-showcase .bi,.bslib-value-box .value-box-showcase .fa,.bslib-value-box .value-box-showcase .fab,.bslib-value-box .value-box-showcase .fas,.bslib-value-box .value-box-showcase .far{opacity:.85;min-width:50px;max-width:125%}.bslib-value-box .value-box-showcase .bi,.bslib-value-box .value-box-showcase .fa,.bslib-value-box .value-box-showcase .fab,.bslib-value-box .value-box-showcase .fas,.bslib-value-box .value-box-showcase .far{font-size:4rem}.bslib-value-box.showcase-top-right .value-box-grid{grid-template-columns:1fr var(---bslib-value-box-showcase-w, 50%)}.bslib-value-box.showcase-top-right .value-box-grid .value-box-showcase{grid-area:right;margin-left:auto;align-self:start;align-items:end;padding-left:0;padding-bottom:0}.bslib-value-box.showcase-top-right .value-box-grid .value-box-area{grid-area:left;align-self:end}.bslib-value-box.showcase-top-right[data-full-screen=true] .value-box-grid{grid-template-columns:auto var(---bslib-value-box-showcase-w-fs, 1fr)}.bslib-value-box.showcase-top-right[data-full-screen=true] .value-box-grid>div{align-self:center}.bslib-value-box.showcase-top-right:not([data-full-screen=true]) .value-box-showcase{margin-top:0}@container bslib-value-box (max-width: 300px){.bslib-value-box.showcase-top-right:not([data-full-screen=true]) .value-box-grid .value-box-showcase{padding-left:1rem}}.bslib-value-box.showcase-left-center .value-box-grid{grid-template-columns:var(---bslib-value-box-showcase-w, 30%) auto}.bslib-value-box.showcase-left-center[data-full-screen=true] .value-box-grid{grid-template-columns:var(---bslib-value-box-showcase-w-fs, 1fr) auto}.bslib-value-box.showcase-left-center:not([data-fill-screen=true]) .value-box-grid .value-box-showcase{grid-area:left}.bslib-value-box.showcase-left-center:not([data-fill-screen=true]) .value-box-grid .value-box-area{grid-area:right}.bslib-value-box.showcase-bottom .value-box-grid{grid-template-columns:1fr;grid-template-rows:1fr var(---bslib-value-box-showcase-h, auto);grid-template-areas:"top" "bottom";overflow:hidden}.bslib-value-box.showcase-bottom .value-box-grid .value-box-showcase{grid-area:bottom;padding:0;margin:0}.bslib-value-box.showcase-bottom .value-box-grid .value-box-area{grid-area:top}.bslib-value-box.showcase-bottom[data-full-screen=true] .value-box-grid{grid-template-rows:1fr var(---bslib-value-box-showcase-h-fs, 2fr)}.bslib-value-box.showcase-bottom[data-full-screen=true] .value-box-grid .value-box-showcase{padding:1rem}[data-bs-theme=dark] .bslib-value-box{--bslib-value-box-shadow: 0 0.5rem 1rem rgb(0 0 0 / 50%)}@media(min-width: 576px){.nav:not(.nav-hidden){display:flex !important;display:-webkit-flex !important}.nav:not(.nav-hidden):not(.nav-stacked):not(.flex-column){float:none !important}.nav:not(.nav-hidden):not(.nav-stacked):not(.flex-column)>.bslib-nav-spacer{margin-left:auto !important}.nav:not(.nav-hidden):not(.nav-stacked):not(.flex-column)>.form-inline{margin-top:auto;margin-bottom:auto}.nav:not(.nav-hidden).nav-stacked{flex-direction:column;-webkit-flex-direction:column;height:100%}.nav:not(.nav-hidden).nav-stacked>.bslib-nav-spacer{margin-top:auto !important}}.bslib-card{overflow:auto}.bslib-card .card-body+.card-body{padding-top:0}.bslib-card .card-body{overflow:auto}.bslib-card .card-body p{margin-top:0}.bslib-card .card-body p:last-child{margin-bottom:0}.bslib-card .card-body{max-height:var(--bslib-card-body-max-height, none)}.bslib-card[data-full-screen=true]>.card-body{max-height:var(--bslib-card-body-max-height-full-screen, none)}.bslib-card .card-header .form-group{margin-bottom:0}.bslib-card .card-header .selectize-control{margin-bottom:0}.bslib-card .card-header .selectize-control .item{margin-right:1.15rem}.bslib-card .card-footer{margin-top:auto}.bslib-card .bslib-navs-card-title{display:flex;flex-wrap:wrap;justify-content:space-between;align-items:center}.bslib-card .bslib-navs-card-title .nav{margin-left:auto}.bslib-card .bslib-sidebar-layout:not([data-bslib-sidebar-border=true]){border:none}.bslib-card .bslib-sidebar-layout:not([data-bslib-sidebar-border-radius=true]){border-top-left-radius:0;border-top-right-radius:0}[data-full-screen=true]{position:fixed;inset:3.5rem 1rem 1rem;height:auto !important;max-height:none !important;width:auto !important;z-index:1070}.bslib-full-screen-enter{display:none;position:absolute;bottom:var(--bslib-full-screen-enter-bottom, 0.2rem);right:var(--bslib-full-screen-enter-right, 0);top:var(--bslib-full-screen-enter-top);left:var(--bslib-full-screen-enter-left);color:var(--bslib-color-fg, var(--bs-card-color));background-color:var(--bslib-color-bg, var(--bs-card-bg, var(--bs-body-bg)));border:var(--bs-card-border-width) solid var(--bslib-color-fg, var(--bs-card-border-color));box-shadow:0 2px 4px rgba(0,0,0,.15);margin:.2rem .4rem;padding:.55rem !important;font-size:.8rem;cursor:pointer;opacity:.7;z-index:1070}.bslib-full-screen-enter:hover{opacity:1}.card[data-full-screen=false]:hover>*>.bslib-full-screen-enter{display:block}.bslib-has-full-screen .card:hover>*>.bslib-full-screen-enter{display:none}@media(max-width: 575.98px){.bslib-full-screen-enter{display:none !important}}.bslib-full-screen-exit{position:relative;top:1.35rem;font-size:.9rem;cursor:pointer;text-decoration:none;display:flex;float:right;margin-right:2.15rem;align-items:center;color:rgba(var(--bs-body-bg-rgb), 0.8)}.bslib-full-screen-exit:hover{color:rgba(var(--bs-body-bg-rgb), 1)}.bslib-full-screen-exit svg{margin-left:.5rem;font-size:1.5rem}#bslib-full-screen-overlay{position:fixed;inset:0;background-color:rgba(var(--bs-body-color-rgb), 0.6);backdrop-filter:blur(2px);-webkit-backdrop-filter:blur(2px);z-index:1069;animation:bslib-full-screen-overlay-enter 400ms cubic-bezier(0.6, 0.02, 0.65, 1) forwards}@keyframes bslib-full-screen-overlay-enter{0%{opacity:0}100%{opacity:1}}.bslib-grid{display:grid !important;gap:var(--bslib-spacer, 1rem);height:var(--bslib-grid-height)}.bslib-grid.grid{grid-template-columns:repeat(var(--bs-columns, 12), minmax(0, 1fr));grid-template-rows:unset;grid-auto-rows:var(--bslib-grid--row-heights);--bslib-grid--row-heights--xs: unset;--bslib-grid--row-heights--sm: unset;--bslib-grid--row-heights--md: unset;--bslib-grid--row-heights--lg: unset;--bslib-grid--row-heights--xl: unset;--bslib-grid--row-heights--xxl: unset}.bslib-grid.grid.bslib-grid--row-heights--xs{--bslib-grid--row-heights: var(--bslib-grid--row-heights--xs)}@media(min-width: 576px){.bslib-grid.grid.bslib-grid--row-heights--sm{--bslib-grid--row-heights: var(--bslib-grid--row-heights--sm)}}@media(min-width: 768px){.bslib-grid.grid.bslib-grid--row-heights--md{--bslib-grid--row-heights: var(--bslib-grid--row-heights--md)}}@media(min-width: 992px){.bslib-grid.grid.bslib-grid--row-heights--lg{--bslib-grid--row-heights: var(--bslib-grid--row-heights--lg)}}@media(min-width: 1200px){.bslib-grid.grid.bslib-grid--row-heights--xl{--bslib-grid--row-heights: var(--bslib-grid--row-heights--xl)}}@media(min-width: 1400px){.bslib-grid.grid.bslib-grid--row-heights--xxl{--bslib-grid--row-heights: var(--bslib-grid--row-heights--xxl)}}.bslib-grid>*>.shiny-input-container{width:100%}.bslib-grid-item{grid-column:auto/span 1}@media(max-width: 767.98px){.bslib-grid-item{grid-column:1/-1}}@media(max-width: 575.98px){.bslib-grid{grid-template-columns:1fr !important;height:var(--bslib-grid-height-mobile)}.bslib-grid.grid{height:unset !important;grid-auto-rows:var(--bslib-grid--row-heights--xs, auto)}}.accordion .accordion-header{font-size:calc(1.29rem + 0.48vw);margin-top:0;margin-bottom:.5rem;font-weight:400;line-height:1.2;color:var(--bs-heading-color);margin-bottom:0}@media(min-width: 1200px){.accordion .accordion-header{font-size:1.65rem}}.accordion .accordion-icon:not(:empty){margin-right:.75rem;display:flex}.accordion .accordion-button:not(.collapsed){box-shadow:none}.accordion .accordion-button:not(.collapsed):focus{box-shadow:var(--bs-accordion-btn-focus-box-shadow)}.navbar+.container-fluid:has(>.tab-content>.tab-pane.active.html-fill-container),.navbar+.container-sm:has(>.tab-content>.tab-pane.active.html-fill-container),.navbar+.container-md:has(>.tab-content>.tab-pane.active.html-fill-container),.navbar+.container-lg:has(>.tab-content>.tab-pane.active.html-fill-container),.navbar+.container-xl:has(>.tab-content>.tab-pane.active.html-fill-container),.navbar+.container-xxl:has(>.tab-content>.tab-pane.active.html-fill-container){padding-left:0;padding-right:0}.navbar+.container-fluid>.tab-content>.tab-pane.active.html-fill-container,.navbar+.container-sm>.tab-content>.tab-pane.active.html-fill-container,.navbar+.container-md>.tab-content>.tab-pane.active.html-fill-container,.navbar+.container-lg>.tab-content>.tab-pane.active.html-fill-container,.navbar+.container-xl>.tab-content>.tab-pane.active.html-fill-container,.navbar+.container-xxl>.tab-content>.tab-pane.active.html-fill-container{padding:var(--bslib-spacer, 1rem);gap:var(--bslib-spacer, 1rem)}.navbar+.container-fluid>.tab-content>.tab-pane.active.html-fill-container:has(>.bslib-sidebar-layout:only-child),.navbar+.container-sm>.tab-content>.tab-pane.active.html-fill-container:has(>.bslib-sidebar-layout:only-child),.navbar+.container-md>.tab-content>.tab-pane.active.html-fill-container:has(>.bslib-sidebar-layout:only-child),.navbar+.container-lg>.tab-content>.tab-pane.active.html-fill-container:has(>.bslib-sidebar-layout:only-child),.navbar+.container-xl>.tab-content>.tab-pane.active.html-fill-container:has(>.bslib-sidebar-layout:only-child),.navbar+.container-xxl>.tab-content>.tab-pane.active.html-fill-container:has(>.bslib-sidebar-layout:only-child){padding:0}.navbar+.container-fluid>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border=true]),.navbar+.container-sm>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border=true]),.navbar+.container-md>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border=true]),.navbar+.container-lg>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border=true]),.navbar+.container-xl>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border=true]),.navbar+.container-xxl>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border=true]){border-left:none;border-right:none;border-bottom:none}.navbar+.container-fluid>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border-radius=true]),.navbar+.container-sm>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border-radius=true]),.navbar+.container-md>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border-radius=true]),.navbar+.container-lg>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border-radius=true]),.navbar+.container-xl>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border-radius=true]),.navbar+.container-xxl>.tab-content>.tab-pane.active.html-fill-container>.bslib-sidebar-layout:only-child:not([data-bslib-sidebar-border-radius=true]){border-radius:0}.navbar+div>.bslib-sidebar-layout{border-top:var(--bslib-sidebar-border)}html{height:100%}.bslib-page-fill{width:100%;height:100%;margin:0;padding:var(--bslib-spacer, 1rem);gap:var(--bslib-spacer, 1rem)}@media(max-width: 575.98px){.bslib-page-fill{height:var(--bslib-page-fill-mobile-height, auto)}}:root{--bslib-page-sidebar-title-bg: #CA225E;--bslib-page-sidebar-title-color: #fff}.bslib-page-title{background-color:var(--bslib-page-sidebar-title-bg);color:var(--bslib-page-sidebar-title-color);font-size:1.25rem;font-weight:300;padding:var(--bslib-spacer, 1rem);padding-left:1.5rem;margin-bottom:0;border-bottom:1px solid #dee2e6}.bslib-sidebar-layout{--bslib-sidebar-transition-duration: 500ms;--bslib-sidebar-transition-easing-x: cubic-bezier(0.8, 0.78, 0.22, 1.07);--bslib-sidebar-border: var(--bs-card-border-width, 1px) solid var(--bs-card-border-color, rgba(0, 0, 0, 0.175));--bslib-sidebar-border-radius: var(--bs-border-radius);--bslib-sidebar-vert-border: var(--bs-card-border-width, 1px) solid var(--bs-card-border-color, rgba(0, 0, 0, 0.175));--bslib-sidebar-bg: rgba(var(--bs-emphasis-color-rgb, 0, 0, 0), 0.05);--bslib-sidebar-fg: var(--bs-emphasis-color, black);--bslib-sidebar-main-fg: var(--bs-card-color, var(--bs-body-color));--bslib-sidebar-main-bg: var(--bs-card-bg, var(--bs-body-bg));--bslib-sidebar-toggle-bg: rgba(var(--bs-emphasis-color-rgb, 0, 0, 0), 0.1);--bslib-sidebar-padding: calc(var(--bslib-spacer) * 1.5);--bslib-sidebar-icon-size: var(--bslib-spacer, 1rem);--bslib-sidebar-icon-button-size: calc(var(--bslib-sidebar-icon-size, 1rem) * 2);--bslib-sidebar-padding-icon: calc(var(--bslib-sidebar-icon-button-size, 2rem) * 1.5);--bslib-collapse-toggle-border-radius: var(--bs-border-radius, 0.25rem);--bslib-collapse-toggle-transform: 0deg;--bslib-sidebar-toggle-transition-easing: cubic-bezier(1, 0, 0, 1);--bslib-collapse-toggle-right-transform: 180deg;--bslib-sidebar-column-main: minmax(0, 1fr);display:grid !important;grid-template-columns:min(100% - var(--bslib-sidebar-icon-size),var(--bslib-sidebar-width, 250px)) var(--bslib-sidebar-column-main);position:relative;transition:grid-template-columns ease-in-out var(--bslib-sidebar-transition-duration);border:var(--bslib-sidebar-border);border-radius:var(--bslib-sidebar-border-radius)}@media(prefers-reduced-motion: reduce){.bslib-sidebar-layout{transition:none}}.bslib-sidebar-layout[data-bslib-sidebar-border=false]{border:none}.bslib-sidebar-layout[data-bslib-sidebar-border-radius=false]{border-radius:initial}.bslib-sidebar-layout>.main,.bslib-sidebar-layout>.sidebar{grid-row:1/2;border-radius:inherit;overflow:auto}.bslib-sidebar-layout>.main{grid-column:2/3;border-top-left-radius:0;border-bottom-left-radius:0;padding:var(--bslib-sidebar-padding);transition:padding var(--bslib-sidebar-transition-easing-x) var(--bslib-sidebar-transition-duration);color:var(--bslib-sidebar-main-fg);background-color:var(--bslib-sidebar-main-bg)}.bslib-sidebar-layout>.sidebar{grid-column:1/2;width:100%;height:100%;border-right:var(--bslib-sidebar-vert-border);border-top-right-radius:0;border-bottom-right-radius:0;color:var(--bslib-sidebar-fg);background-color:var(--bslib-sidebar-bg);backdrop-filter:blur(5px)}.bslib-sidebar-layout>.sidebar>.sidebar-content{display:flex;flex-direction:column;gap:var(--bslib-spacer, 1rem);padding:var(--bslib-sidebar-padding);padding-top:var(--bslib-sidebar-padding-icon)}.bslib-sidebar-layout>.sidebar>.sidebar-content>:last-child:not(.sidebar-title){margin-bottom:0}.bslib-sidebar-layout>.sidebar>.sidebar-content>.accordion{margin-left:calc(-1*var(--bslib-sidebar-padding));margin-right:calc(-1*var(--bslib-sidebar-padding))}.bslib-sidebar-layout>.sidebar>.sidebar-content>.accordion:last-child{margin-bottom:calc(-1*var(--bslib-sidebar-padding))}.bslib-sidebar-layout>.sidebar>.sidebar-content>.accordion:not(:last-child){margin-bottom:1rem}.bslib-sidebar-layout>.sidebar>.sidebar-content>.accordion .accordion-body{display:flex;flex-direction:column}.bslib-sidebar-layout>.sidebar>.sidebar-content>.accordion:not(:first-child) .accordion-item:first-child{border-top:var(--bs-accordion-border-width) solid var(--bs-accordion-border-color)}.bslib-sidebar-layout>.sidebar>.sidebar-content>.accordion:not(:last-child) .accordion-item:last-child{border-bottom:var(--bs-accordion-border-width) solid var(--bs-accordion-border-color)}.bslib-sidebar-layout>.sidebar>.sidebar-content.has-accordion>.sidebar-title{border-bottom:none;padding-bottom:0}.bslib-sidebar-layout>.sidebar .shiny-input-container{width:100%}.bslib-sidebar-layout[data-bslib-sidebar-open=always]>.sidebar>.sidebar-content{padding-top:var(--bslib-sidebar-padding)}.bslib-sidebar-layout>.collapse-toggle{grid-row:1/2;grid-column:1/2;display:inline-flex;align-items:center;position:absolute;right:calc(var(--bslib-sidebar-icon-size));top:calc(var(--bslib-sidebar-icon-size, 1rem)/2);border:none;border-radius:var(--bslib-collapse-toggle-border-radius);height:var(--bslib-sidebar-icon-button-size, 2rem);width:var(--bslib-sidebar-icon-button-size, 2rem);display:flex;align-items:center;justify-content:center;padding:0;color:var(--bslib-sidebar-fg);background-color:unset;transition:color var(--bslib-sidebar-transition-easing-x) var(--bslib-sidebar-transition-duration),top var(--bslib-sidebar-transition-easing-x) var(--bslib-sidebar-transition-duration),right var(--bslib-sidebar-transition-easing-x) var(--bslib-sidebar-transition-duration),left var(--bslib-sidebar-transition-easing-x) var(--bslib-sidebar-transition-duration)}.bslib-sidebar-layout>.collapse-toggle:hover{background-color:var(--bslib-sidebar-toggle-bg)}.bslib-sidebar-layout>.collapse-toggle>.collapse-icon{opacity:.8;width:var(--bslib-sidebar-icon-size);height:var(--bslib-sidebar-icon-size);transform:rotateY(var(--bslib-collapse-toggle-transform));transition:transform var(--bslib-sidebar-toggle-transition-easing) var(--bslib-sidebar-transition-duration)}.bslib-sidebar-layout>.collapse-toggle:hover>.collapse-icon{opacity:1}.bslib-sidebar-layout .sidebar-title{font-size:1.25rem;line-height:1.25;margin-top:0;margin-bottom:1rem;padding-bottom:1rem;border-bottom:var(--bslib-sidebar-border)}.bslib-sidebar-layout.sidebar-right{grid-template-columns:var(--bslib-sidebar-column-main) min(100% - var(--bslib-sidebar-icon-size),var(--bslib-sidebar-width, 250px))}.bslib-sidebar-layout.sidebar-right>.main{grid-column:1/2;border-top-right-radius:0;border-bottom-right-radius:0;border-top-left-radius:inherit;border-bottom-left-radius:inherit}.bslib-sidebar-layout.sidebar-right>.sidebar{grid-column:2/3;border-right:none;border-left:var(--bslib-sidebar-vert-border);border-top-left-radius:0;border-bottom-left-radius:0}.bslib-sidebar-layout.sidebar-right>.collapse-toggle{grid-column:2/3;left:var(--bslib-sidebar-icon-size);right:unset;border:var(--bslib-collapse-toggle-border)}.bslib-sidebar-layout.sidebar-right>.collapse-toggle>.collapse-icon{transform:rotateY(var(--bslib-collapse-toggle-right-transform))}.bslib-sidebar-layout.sidebar-collapsed{--bslib-collapse-toggle-transform: 180deg;--bslib-collapse-toggle-right-transform: 0deg;--bslib-sidebar-vert-border: none;grid-template-columns:0 minmax(0, 1fr)}.bslib-sidebar-layout.sidebar-collapsed.sidebar-right{grid-template-columns:minmax(0, 1fr) 0}.bslib-sidebar-layout.sidebar-collapsed:not(.transitioning)>.sidebar>*{display:none}.bslib-sidebar-layout.sidebar-collapsed>.main{border-radius:inherit}.bslib-sidebar-layout.sidebar-collapsed:not(.sidebar-right)>.main{padding-left:var(--bslib-sidebar-padding-icon)}.bslib-sidebar-layout.sidebar-collapsed.sidebar-right>.main{padding-right:var(--bslib-sidebar-padding-icon)}.bslib-sidebar-layout.sidebar-collapsed>.collapse-toggle{color:var(--bslib-sidebar-main-fg);top:calc(var(--bslib-sidebar-overlap-counter, 0)*(var(--bslib-sidebar-icon-size) + var(--bslib-sidebar-padding)) + var(--bslib-sidebar-icon-size, 1rem)/2);right:calc(-2.5*var(--bslib-sidebar-icon-size) - var(--bs-card-border-width, 1px))}.bslib-sidebar-layout.sidebar-collapsed.sidebar-right>.collapse-toggle{left:calc(-2.5*var(--bslib-sidebar-icon-size) - var(--bs-card-border-width, 1px));right:unset}@media(min-width: 576px){.bslib-sidebar-layout.transitioning>.sidebar>.sidebar-content{display:none}}@media(max-width: 575.98px){.bslib-sidebar-layout[data-bslib-sidebar-open=desktop]{--bslib-sidebar-js-init-collapsed: true}.bslib-sidebar-layout>.sidebar,.bslib-sidebar-layout.sidebar-right>.sidebar{border:none}.bslib-sidebar-layout>.main,.bslib-sidebar-layout.sidebar-right>.main{grid-column:1/3}.bslib-sidebar-layout[data-bslib-sidebar-open=always]{display:block !important}.bslib-sidebar-layout[data-bslib-sidebar-open=always]>.sidebar{max-height:var(--bslib-sidebar-max-height-mobile);overflow-y:auto;border-top:var(--bslib-sidebar-vert-border)}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always]){grid-template-columns:100% 0}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always]):not(.sidebar-collapsed)>.sidebar{z-index:1}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always]):not(.sidebar-collapsed)>.collapse-toggle{z-index:1}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always]).sidebar-right{grid-template-columns:0 100%}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always]).sidebar-collapsed{grid-template-columns:0 100%}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always]).sidebar-collapsed.sidebar-right{grid-template-columns:100% 0}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always]):not(.sidebar-right)>.main{padding-left:var(--bslib-sidebar-padding-icon)}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always]).sidebar-right>.main{padding-right:var(--bslib-sidebar-padding-icon)}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always])>.main{opacity:0;transition:opacity var(--bslib-sidebar-transition-easing-x) var(--bslib-sidebar-transition-duration)}.bslib-sidebar-layout:not([data-bslib-sidebar-open=always]).sidebar-collapsed>.main{opacity:1}}.html-fill-container{display:flex;flex-direction:column;min-height:0;min-width:0}.html-fill-container>.html-fill-item{flex:1 1 auto;min-height:0;min-width:0}.html-fill-container>:not(.html-fill-item){flex:0 0 auto}.quarto-container{min-height:calc(100vh - 132px)}body.hypothesis-enabled #quarto-header{margin-right:16px}footer.footer .nav-footer,#quarto-header>nav{padding-left:1em;padding-right:1em}footer.footer div.nav-footer p:first-child{margin-top:0}footer.footer div.nav-footer p:last-child{margin-bottom:0}#quarto-content>*{padding-top:14px}#quarto-content>#quarto-sidebar-glass{padding-top:0px}@media(max-width: 991.98px){#quarto-content>*{padding-top:0}#quarto-content .subtitle{padding-top:14px}#quarto-content section:first-of-type h2:first-of-type,#quarto-content section:first-of-type .h2:first-of-type{margin-top:1rem}}.headroom-target,header.headroom{will-change:transform;transition:position 200ms linear;transition:all 200ms linear}header.headroom--pinned{transform:translateY(0%)}header.headroom--unpinned{transform:translateY(-100%)}.navbar-container{width:100%}.navbar-brand{overflow:hidden;text-overflow:ellipsis}.navbar-brand-container{max-width:calc(100% - 115px);min-width:0;display:flex;align-items:center}@media(min-width: 992px){.navbar-brand-container{margin-right:1em}}.navbar-brand.navbar-brand-logo{margin-right:4px;display:inline-flex}.navbar-toggler{flex-basis:content;flex-shrink:0}.navbar .navbar-brand-container{order:2}.navbar .navbar-toggler{order:1}.navbar .navbar-container>.navbar-nav{order:20}.navbar .navbar-container>.navbar-brand-container{margin-left:0 !important;margin-right:0 !important}.navbar .navbar-collapse{order:20}.navbar #quarto-search{order:4;margin-left:auto}.navbar .navbar-toggler{margin-right:.5em}.navbar-collapse .quarto-navbar-tools{margin-left:.5em}.navbar-logo{max-height:24px;width:auto;padding-right:4px}nav .nav-item:not(.compact){padding-top:1px}nav .nav-link i,nav .dropdown-item i{padding-right:1px}.navbar-expand-lg .navbar-nav .nav-link{padding-left:.6rem;padding-right:.6rem}nav .nav-item.compact .nav-link{padding-left:.5rem;padding-right:.5rem;font-size:1.1rem}.navbar .quarto-navbar-tools{order:3}.navbar .quarto-navbar-tools div.dropdown{display:inline-block}.navbar .quarto-navbar-tools .quarto-navigation-tool{color:#fefdfd}.navbar .quarto-navbar-tools .quarto-navigation-tool:hover{color:#fdfdff}.navbar-nav .dropdown-menu{min-width:220px;font-size:.9rem}.navbar .navbar-nav .nav-link.dropdown-toggle::after{opacity:.75;vertical-align:.175em}.navbar ul.dropdown-menu{padding-top:0;padding-bottom:0}.navbar .dropdown-header{text-transform:uppercase;font-size:.8rem;padding:0 .5rem}.navbar .dropdown-item{padding:.4rem .5rem}.navbar .dropdown-item>i.bi{margin-left:.1rem;margin-right:.25em}.sidebar #quarto-search{margin-top:-1px}.sidebar #quarto-search svg.aa-SubmitIcon{width:16px;height:16px}.sidebar-navigation a{color:inherit}.sidebar-title{margin-top:.25rem;padding-bottom:.5rem;font-size:1.3rem;line-height:1.6rem;visibility:visible}.sidebar-title>a{font-size:inherit;text-decoration:none}.sidebar-title .sidebar-tools-main{margin-top:-6px}@media(max-width: 991.98px){#quarto-sidebar div.sidebar-header{padding-top:.2em}}.sidebar-header-stacked .sidebar-title{margin-top:.6rem}.sidebar-logo{max-width:90%;padding-bottom:.5rem}.sidebar-logo-link{text-decoration:none}.sidebar-navigation li a{text-decoration:none}.sidebar-navigation .quarto-navigation-tool{opacity:.7;font-size:.875rem}#quarto-sidebar>nav>.sidebar-tools-main{margin-left:14px}.sidebar-tools-main{display:inline-flex;margin-left:0px;order:2}.sidebar-tools-main:not(.tools-wide){vertical-align:middle}.sidebar-navigation .quarto-navigation-tool.dropdown-toggle::after{display:none}.sidebar.sidebar-navigation>*{padding-top:1em}.sidebar-item{margin-bottom:.2em;line-height:1rem;margin-top:.4rem}.sidebar-section{padding-left:.5em;padding-bottom:.2em}.sidebar-item .sidebar-item-container{display:flex;justify-content:space-between;cursor:pointer}.sidebar-item-toggle:hover{cursor:pointer}.sidebar-item .sidebar-item-toggle .bi{font-size:.7rem;text-align:center}.sidebar-item .sidebar-item-toggle .bi-chevron-right::before{transition:transform 200ms ease}.sidebar-item .sidebar-item-toggle[aria-expanded=false] .bi-chevron-right::before{transform:none}.sidebar-item .sidebar-item-toggle[aria-expanded=true] .bi-chevron-right::before{transform:rotate(90deg)}.sidebar-item-text{width:100%}.sidebar-navigation .sidebar-divider{margin-left:0;margin-right:0;margin-top:.5rem;margin-bottom:.5rem}@media(max-width: 991.98px){.quarto-secondary-nav{display:block}.quarto-secondary-nav button.quarto-search-button{padding-right:0em;padding-left:2em}.quarto-secondary-nav button.quarto-btn-toggle{margin-left:-0.75rem;margin-right:.15rem}.quarto-secondary-nav nav.quarto-title-breadcrumbs{display:none}.quarto-secondary-nav nav.quarto-page-breadcrumbs{display:flex;align-items:center;padding-right:1em;margin-left:-0.25em}.quarto-secondary-nav nav.quarto-page-breadcrumbs a{text-decoration:none}.quarto-secondary-nav nav.quarto-page-breadcrumbs ol.breadcrumb{margin-bottom:0}}@media(min-width: 992px){.quarto-secondary-nav{display:none}}.quarto-title-breadcrumbs .breadcrumb{margin-bottom:.5em;font-size:.9rem}.quarto-title-breadcrumbs .breadcrumb li:last-of-type a{color:#6c757d}.quarto-secondary-nav .quarto-btn-toggle{color:#595959}.quarto-secondary-nav[aria-expanded=false] .quarto-btn-toggle .bi-chevron-right::before{transform:none}.quarto-secondary-nav[aria-expanded=true] .quarto-btn-toggle .bi-chevron-right::before{transform:rotate(90deg)}.quarto-secondary-nav .quarto-btn-toggle .bi-chevron-right::before{transition:transform 200ms ease}.quarto-secondary-nav{cursor:pointer}.no-decor{text-decoration:none}.quarto-secondary-nav-title{margin-top:.3em;color:#595959;padding-top:4px}.quarto-secondary-nav nav.quarto-page-breadcrumbs{color:#595959}.quarto-secondary-nav nav.quarto-page-breadcrumbs a{color:#595959}.quarto-secondary-nav nav.quarto-page-breadcrumbs a:hover{color:rgba(33,81,191,.8)}.quarto-secondary-nav nav.quarto-page-breadcrumbs .breadcrumb-item::before{color:#8c8c8c}.breadcrumb-item{line-height:1.2rem}div.sidebar-item-container{color:#595959}div.sidebar-item-container:hover,div.sidebar-item-container:focus{color:rgba(33,81,191,.8)}div.sidebar-item-container.disabled{color:rgba(89,89,89,.75)}div.sidebar-item-container .active,div.sidebar-item-container .show>.nav-link,div.sidebar-item-container .sidebar-link>code{color:#2151bf}div.sidebar.sidebar-navigation.rollup.quarto-sidebar-toggle-contents,nav.sidebar.sidebar-navigation:not(.rollup){background-color:#fff}@media(max-width: 991.98px){.sidebar-navigation .sidebar-item a,.nav-page .nav-page-text,.sidebar-navigation{font-size:1rem}.sidebar-navigation ul.sidebar-section.depth1 .sidebar-section-item{font-size:1.1rem}.sidebar-logo{display:none}.sidebar.sidebar-navigation{position:static;border-bottom:1px solid #dee2e6}.sidebar.sidebar-navigation.collapsing{position:fixed;z-index:1000}.sidebar.sidebar-navigation.show{position:fixed;z-index:1000}.sidebar.sidebar-navigation{min-height:100%}nav.quarto-secondary-nav{background-color:#fff;border-bottom:1px solid #dee2e6}.quarto-banner nav.quarto-secondary-nav{background-color:#ca225e;color:#fefdfd;border-top:1px solid #dee2e6}.sidebar .sidebar-footer{visibility:visible;padding-top:1rem;position:inherit}.sidebar-tools-collapse{display:block}}#quarto-sidebar{transition:width .15s ease-in}#quarto-sidebar>*{padding-right:1em}@media(max-width: 991.98px){#quarto-sidebar .sidebar-menu-container{white-space:nowrap;min-width:225px}#quarto-sidebar.show{transition:width .15s ease-out}}@media(min-width: 992px){#quarto-sidebar{display:flex;flex-direction:column}.nav-page .nav-page-text,.sidebar-navigation .sidebar-section .sidebar-item{font-size:.875rem}.sidebar-navigation .sidebar-item{font-size:.925rem}.sidebar.sidebar-navigation{display:block;position:sticky}.sidebar-search{width:100%}.sidebar .sidebar-footer{visibility:visible}}@media(min-width: 992px){#quarto-sidebar-glass{display:none}}@media(max-width: 991.98px){#quarto-sidebar-glass{position:fixed;top:0;bottom:0;left:0;right:0;background-color:rgba(255,255,255,0);transition:background-color .15s ease-in;z-index:-1}#quarto-sidebar-glass.collapsing{z-index:1000}#quarto-sidebar-glass.show{transition:background-color .15s ease-out;background-color:rgba(102,102,102,.4);z-index:1000}}.sidebar .sidebar-footer{padding:.5rem 1rem;align-self:flex-end;color:#6c757d;width:100%}.quarto-page-breadcrumbs .breadcrumb-item+.breadcrumb-item,.quarto-page-breadcrumbs .breadcrumb-item{padding-right:.33em;padding-left:0}.quarto-page-breadcrumbs .breadcrumb-item::before{padding-right:.33em}.quarto-sidebar-footer{font-size:.875em}.sidebar-section .bi-chevron-right{vertical-align:middle}.sidebar-section .bi-chevron-right::before{font-size:.9em}.notransition{-webkit-transition:none !important;-moz-transition:none !important;-o-transition:none !important;transition:none !important}.btn:focus:not(:focus-visible){box-shadow:none}.page-navigation{display:flex;justify-content:space-between}.nav-page{padding-bottom:.75em}.nav-page .bi{font-size:1.8rem;vertical-align:middle}.nav-page .nav-page-text{padding-left:.25em;padding-right:.25em}.nav-page a{color:#6c757d;text-decoration:none;display:flex;align-items:center}.nav-page a:hover{color:#1f4eb6}.nav-footer .toc-actions{padding-bottom:.5em;padding-top:.5em}.nav-footer .toc-actions a,.nav-footer .toc-actions a:hover{text-decoration:none}.nav-footer .toc-actions ul{display:flex;list-style:none}.nav-footer .toc-actions ul :first-child{margin-left:auto}.nav-footer .toc-actions ul :last-child{margin-right:auto}.nav-footer .toc-actions ul li{padding-right:1.5em}.nav-footer .toc-actions ul li i.bi{padding-right:.4em}.nav-footer .toc-actions ul li:last-of-type{padding-right:0}.nav-footer{display:flex;flex-direction:row;flex-wrap:wrap;justify-content:space-between;align-items:baseline;text-align:center;padding-top:.5rem;padding-bottom:.5rem;background-color:#fff}body.nav-fixed{padding-top:64px}.nav-footer-contents{color:#6c757d;margin-top:.25rem}.nav-footer{min-height:3.5em;color:#757575}.nav-footer a{color:#757575}.nav-footer .nav-footer-left{font-size:.825em}.nav-footer .nav-footer-center{font-size:.825em}.nav-footer .nav-footer-right{font-size:.825em}.nav-footer-left .footer-items,.nav-footer-center .footer-items,.nav-footer-right .footer-items{display:inline-flex;padding-top:.3em;padding-bottom:.3em;margin-bottom:0em}.nav-footer-left .footer-items .nav-link,.nav-footer-center .footer-items .nav-link,.nav-footer-right .footer-items .nav-link{padding-left:.6em;padding-right:.6em}@media(min-width: 768px){.nav-footer-left{flex:1 1 0px;text-align:left}}@media(max-width: 575.98px){.nav-footer-left{margin-bottom:1em;flex:100%}}@media(min-width: 768px){.nav-footer-right{flex:1 1 0px;text-align:right}}@media(max-width: 575.98px){.nav-footer-right{margin-bottom:1em;flex:100%}}.nav-footer-center{text-align:center;min-height:3em}@media(min-width: 768px){.nav-footer-center{flex:1 1 0px}}.nav-footer-center .footer-items{justify-content:center}@media(max-width: 767.98px){.nav-footer-center{margin-bottom:1em;flex:100%}}@media(max-width: 767.98px){.nav-footer-center{margin-top:3em;order:10}}.navbar .quarto-reader-toggle.reader .quarto-reader-toggle-btn{background-color:#fefdfd;border-radius:3px}@media(max-width: 991.98px){.quarto-reader-toggle{display:none}}.quarto-reader-toggle.reader.quarto-navigation-tool .quarto-reader-toggle-btn{background-color:#595959;border-radius:3px}.quarto-reader-toggle .quarto-reader-toggle-btn{display:inline-flex;padding-left:.2em;padding-right:.2em;margin-left:-0.2em;margin-right:-0.2em;text-align:center}.navbar .quarto-reader-toggle:not(.reader) .bi::before{background-image:url('data:image/svg+xml,')}.navbar .quarto-reader-toggle.reader .bi::before{background-image:url('data:image/svg+xml,')}.sidebar-navigation .quarto-reader-toggle:not(.reader) .bi::before{background-image:url('data:image/svg+xml,')}.sidebar-navigation .quarto-reader-toggle.reader .bi::before{background-image:url('data:image/svg+xml,')}#quarto-back-to-top{display:none;position:fixed;bottom:50px;background-color:#fff;border-radius:.25rem;box-shadow:0 .2rem .5rem #6c757d,0 0 .05rem #6c757d;color:#6c757d;text-decoration:none;font-size:.9em;text-align:center;left:50%;padding:.4rem .8rem;transform:translate(-50%, 0)}#quarto-announcement{padding:.5em;display:flex;justify-content:space-between;margin-bottom:0;font-size:.9em}#quarto-announcement .quarto-announcement-content{margin-right:auto}#quarto-announcement .quarto-announcement-content p{margin-bottom:0}#quarto-announcement .quarto-announcement-icon{margin-right:.5em;font-size:1.2em;margin-top:-0.15em}#quarto-announcement .quarto-announcement-action{cursor:pointer}.aa-DetachedSearchButtonQuery{display:none}.aa-DetachedOverlay ul.aa-List,#quarto-search-results ul.aa-List{list-style:none;padding-left:0}.aa-DetachedOverlay .aa-Panel,#quarto-search-results .aa-Panel{background-color:#fff;position:absolute;z-index:2000}#quarto-search-results .aa-Panel{max-width:400px}#quarto-search input{font-size:.925rem}@media(min-width: 992px){.navbar #quarto-search{margin-left:.25rem;order:999}}.navbar.navbar-expand-sm #quarto-search,.navbar.navbar-expand-md #quarto-search{order:999}@media(min-width: 992px){.navbar .quarto-navbar-tools{order:900}}@media(min-width: 992px){.navbar .quarto-navbar-tools.tools-end{margin-left:auto !important}}@media(max-width: 991.98px){#quarto-sidebar .sidebar-search{display:none}}#quarto-sidebar .sidebar-search .aa-Autocomplete{width:100%}.navbar .aa-Autocomplete .aa-Form{width:180px}.navbar #quarto-search.type-overlay .aa-Autocomplete{width:40px}.navbar #quarto-search.type-overlay .aa-Autocomplete .aa-Form{background-color:inherit;border:none}.navbar #quarto-search.type-overlay .aa-Autocomplete .aa-Form:focus-within{box-shadow:none;outline:none}.navbar #quarto-search.type-overlay .aa-Autocomplete .aa-Form .aa-InputWrapper{display:none}.navbar #quarto-search.type-overlay .aa-Autocomplete .aa-Form .aa-InputWrapper:focus-within{display:inherit}.navbar #quarto-search.type-overlay .aa-Autocomplete .aa-Form .aa-Label svg,.navbar #quarto-search.type-overlay .aa-Autocomplete .aa-Form .aa-LoadingIndicator svg{width:26px;height:26px;color:#fefdfd;opacity:1}.navbar #quarto-search.type-overlay .aa-Autocomplete svg.aa-SubmitIcon{width:26px;height:26px;color:#fefdfd;opacity:1}.aa-Autocomplete .aa-Form,.aa-DetachedFormContainer .aa-Form{align-items:center;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;color:#343a40;display:flex;line-height:1em;margin:0;position:relative;width:100%}.aa-Autocomplete .aa-Form:focus-within,.aa-DetachedFormContainer .aa-Form:focus-within{box-shadow:rgba(39,128,227,.6) 0 0 0 1px;outline:currentColor none medium}.aa-Autocomplete .aa-Form .aa-InputWrapperPrefix,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperPrefix{align-items:center;display:flex;flex-shrink:0;order:1}.aa-Autocomplete .aa-Form .aa-InputWrapperPrefix .aa-Label,.aa-Autocomplete .aa-Form .aa-InputWrapperPrefix .aa-LoadingIndicator,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperPrefix .aa-Label,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperPrefix .aa-LoadingIndicator{cursor:initial;flex-shrink:0;padding:0;text-align:left}.aa-Autocomplete .aa-Form .aa-InputWrapperPrefix .aa-Label svg,.aa-Autocomplete .aa-Form .aa-InputWrapperPrefix .aa-LoadingIndicator svg,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperPrefix .aa-Label svg,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperPrefix .aa-LoadingIndicator svg{color:#343a40;opacity:.5}.aa-Autocomplete .aa-Form .aa-InputWrapperPrefix .aa-SubmitButton,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperPrefix .aa-SubmitButton{appearance:none;background:none;border:0;margin:0}.aa-Autocomplete .aa-Form .aa-InputWrapperPrefix .aa-LoadingIndicator,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperPrefix .aa-LoadingIndicator{align-items:center;display:flex;justify-content:center}.aa-Autocomplete .aa-Form .aa-InputWrapperPrefix .aa-LoadingIndicator[hidden],.aa-DetachedFormContainer .aa-Form .aa-InputWrapperPrefix .aa-LoadingIndicator[hidden]{display:none}.aa-Autocomplete .aa-Form .aa-InputWrapper,.aa-DetachedFormContainer .aa-Form .aa-InputWrapper{order:3;position:relative;width:100%}.aa-Autocomplete .aa-Form .aa-InputWrapper .aa-Input,.aa-DetachedFormContainer .aa-Form .aa-InputWrapper .aa-Input{appearance:none;background:none;border:0;color:#343a40;font:inherit;height:calc(1.5em + .1rem + 2px);padding:0;width:100%}.aa-Autocomplete .aa-Form .aa-InputWrapper .aa-Input::placeholder,.aa-DetachedFormContainer .aa-Form .aa-InputWrapper .aa-Input::placeholder{color:#343a40;opacity:.8}.aa-Autocomplete .aa-Form .aa-InputWrapper .aa-Input:focus,.aa-DetachedFormContainer .aa-Form .aa-InputWrapper .aa-Input:focus{border-color:none;box-shadow:none;outline:none}.aa-Autocomplete .aa-Form .aa-InputWrapper .aa-Input::-webkit-search-decoration,.aa-Autocomplete .aa-Form .aa-InputWrapper .aa-Input::-webkit-search-cancel-button,.aa-Autocomplete .aa-Form .aa-InputWrapper .aa-Input::-webkit-search-results-button,.aa-Autocomplete .aa-Form .aa-InputWrapper .aa-Input::-webkit-search-results-decoration,.aa-DetachedFormContainer .aa-Form .aa-InputWrapper .aa-Input::-webkit-search-decoration,.aa-DetachedFormContainer .aa-Form .aa-InputWrapper .aa-Input::-webkit-search-cancel-button,.aa-DetachedFormContainer .aa-Form .aa-InputWrapper .aa-Input::-webkit-search-results-button,.aa-DetachedFormContainer .aa-Form .aa-InputWrapper .aa-Input::-webkit-search-results-decoration{display:none}.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix{align-items:center;display:flex;order:4}.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix .aa-ClearButton,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix .aa-ClearButton{align-items:center;background:none;border:0;color:#343a40;opacity:.8;cursor:pointer;display:flex;margin:0;width:calc(1.5em + .1rem + 2px)}.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix .aa-ClearButton:hover,.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix .aa-ClearButton:focus,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix .aa-ClearButton:hover,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix .aa-ClearButton:focus{color:#343a40;opacity:.8}.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix .aa-ClearButton[hidden],.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix .aa-ClearButton[hidden]{display:none}.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix .aa-ClearButton svg,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix .aa-ClearButton svg{width:calc(1.5em + 0.75rem + calc(1px * 2))}.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix .aa-CopyButton,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix .aa-CopyButton{border:none;align-items:center;background:none;color:#343a40;opacity:.4;font-size:.7rem;cursor:pointer;display:none;margin:0;width:calc(1em + .1rem + 2px)}.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix .aa-CopyButton:hover,.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix .aa-CopyButton:focus,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix .aa-CopyButton:hover,.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix .aa-CopyButton:focus{color:#343a40;opacity:.8}.aa-Autocomplete .aa-Form .aa-InputWrapperSuffix .aa-CopyButton[hidden],.aa-DetachedFormContainer .aa-Form .aa-InputWrapperSuffix .aa-CopyButton[hidden]{display:none}.aa-PanelLayout:empty{display:none}.quarto-search-no-results.no-query{display:none}.aa-Source:has(.no-query){display:none}#quarto-search-results .aa-Panel{border:solid #dee2e6 1px}#quarto-search-results .aa-SourceNoResults{width:398px}.aa-DetachedOverlay .aa-Panel,#quarto-search-results .aa-Panel{max-height:65vh;overflow-y:auto;font-size:.925rem}.aa-DetachedOverlay .aa-SourceNoResults,#quarto-search-results .aa-SourceNoResults{height:60px;display:flex;justify-content:center;align-items:center}.aa-DetachedOverlay .search-error,#quarto-search-results .search-error{padding-top:10px;padding-left:20px;padding-right:20px;cursor:default}.aa-DetachedOverlay .search-error .search-error-title,#quarto-search-results .search-error .search-error-title{font-size:1.1rem;margin-bottom:.5rem}.aa-DetachedOverlay .search-error .search-error-title .search-error-icon,#quarto-search-results .search-error .search-error-title .search-error-icon{margin-right:8px}.aa-DetachedOverlay .search-error .search-error-text,#quarto-search-results .search-error .search-error-text{font-weight:300}.aa-DetachedOverlay .search-result-text,#quarto-search-results .search-result-text{font-weight:300;overflow:hidden;text-overflow:ellipsis;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;line-height:1.2rem;max-height:2.4rem}.aa-DetachedOverlay .aa-SourceHeader .search-result-header,#quarto-search-results .aa-SourceHeader .search-result-header{font-size:.875rem;background-color:#f2f2f2;padding-left:14px;padding-bottom:4px;padding-top:4px}.aa-DetachedOverlay .aa-SourceHeader .search-result-header-no-results,#quarto-search-results .aa-SourceHeader .search-result-header-no-results{display:none}.aa-DetachedOverlay .aa-SourceFooter .algolia-search-logo,#quarto-search-results .aa-SourceFooter .algolia-search-logo{width:110px;opacity:.85;margin:8px;float:right}.aa-DetachedOverlay .search-result-section,#quarto-search-results .search-result-section{font-size:.925em}.aa-DetachedOverlay a.search-result-link,#quarto-search-results a.search-result-link{color:inherit;text-decoration:none}.aa-DetachedOverlay li.aa-Item[aria-selected=true] .search-item,#quarto-search-results li.aa-Item[aria-selected=true] .search-item{background-color:#2780e3}.aa-DetachedOverlay li.aa-Item[aria-selected=true] .search-item.search-result-more,.aa-DetachedOverlay li.aa-Item[aria-selected=true] .search-item .search-result-section,.aa-DetachedOverlay li.aa-Item[aria-selected=true] .search-item .search-result-text,.aa-DetachedOverlay li.aa-Item[aria-selected=true] .search-item .search-result-title-container,.aa-DetachedOverlay li.aa-Item[aria-selected=true] .search-item .search-result-text-container,#quarto-search-results li.aa-Item[aria-selected=true] .search-item.search-result-more,#quarto-search-results li.aa-Item[aria-selected=true] .search-item .search-result-section,#quarto-search-results li.aa-Item[aria-selected=true] .search-item .search-result-text,#quarto-search-results li.aa-Item[aria-selected=true] .search-item .search-result-title-container,#quarto-search-results li.aa-Item[aria-selected=true] .search-item .search-result-text-container{color:#fff;background-color:#2780e3}.aa-DetachedOverlay li.aa-Item[aria-selected=true] .search-item mark.search-match,.aa-DetachedOverlay li.aa-Item[aria-selected=true] .search-item .search-match.mark,#quarto-search-results li.aa-Item[aria-selected=true] .search-item mark.search-match,#quarto-search-results li.aa-Item[aria-selected=true] .search-item .search-match.mark{color:#fff;background-color:#4b95e8}.aa-DetachedOverlay li.aa-Item[aria-selected=false] .search-item,#quarto-search-results li.aa-Item[aria-selected=false] .search-item{background-color:#fff}.aa-DetachedOverlay li.aa-Item[aria-selected=false] .search-item.search-result-more,.aa-DetachedOverlay li.aa-Item[aria-selected=false] .search-item .search-result-section,.aa-DetachedOverlay li.aa-Item[aria-selected=false] .search-item .search-result-text,.aa-DetachedOverlay li.aa-Item[aria-selected=false] .search-item .search-result-title-container,.aa-DetachedOverlay li.aa-Item[aria-selected=false] .search-item .search-result-text-container,#quarto-search-results li.aa-Item[aria-selected=false] .search-item.search-result-more,#quarto-search-results li.aa-Item[aria-selected=false] .search-item .search-result-section,#quarto-search-results li.aa-Item[aria-selected=false] .search-item .search-result-text,#quarto-search-results li.aa-Item[aria-selected=false] .search-item .search-result-title-container,#quarto-search-results li.aa-Item[aria-selected=false] .search-item .search-result-text-container{color:#343a40}.aa-DetachedOverlay li.aa-Item[aria-selected=false] .search-item mark.search-match,.aa-DetachedOverlay li.aa-Item[aria-selected=false] .search-item .search-match.mark,#quarto-search-results li.aa-Item[aria-selected=false] .search-item mark.search-match,#quarto-search-results li.aa-Item[aria-selected=false] .search-item .search-match.mark{color:inherit;background-color:#e5effc}.aa-DetachedOverlay .aa-Item .search-result-doc:not(.document-selectable) .search-result-title-container,#quarto-search-results .aa-Item .search-result-doc:not(.document-selectable) .search-result-title-container{background-color:#fff;color:#343a40}.aa-DetachedOverlay .aa-Item .search-result-doc:not(.document-selectable) .search-result-text-container,#quarto-search-results .aa-Item .search-result-doc:not(.document-selectable) .search-result-text-container{padding-top:0px}.aa-DetachedOverlay li.aa-Item .search-result-doc.document-selectable .search-result-text-container,#quarto-search-results li.aa-Item .search-result-doc.document-selectable .search-result-text-container{margin-top:-4px}.aa-DetachedOverlay .aa-Item,#quarto-search-results .aa-Item{cursor:pointer}.aa-DetachedOverlay .aa-Item .search-item,#quarto-search-results .aa-Item .search-item{border-left:none;border-right:none;border-top:none;background-color:#fff;border-color:#dee2e6;color:#343a40}.aa-DetachedOverlay .aa-Item .search-item p,#quarto-search-results .aa-Item .search-item p{margin-top:0;margin-bottom:0}.aa-DetachedOverlay .aa-Item .search-item i.bi,#quarto-search-results .aa-Item .search-item i.bi{padding-left:8px;padding-right:8px;font-size:1.3em}.aa-DetachedOverlay .aa-Item .search-item .search-result-title,#quarto-search-results .aa-Item .search-item .search-result-title{margin-top:.3em;margin-bottom:0em}.aa-DetachedOverlay .aa-Item .search-item .search-result-crumbs,#quarto-search-results .aa-Item .search-item .search-result-crumbs{white-space:nowrap;text-overflow:ellipsis;font-size:.8em;font-weight:300;margin-right:1em}.aa-DetachedOverlay .aa-Item .search-item .search-result-crumbs:not(.search-result-crumbs-wrap),#quarto-search-results .aa-Item .search-item .search-result-crumbs:not(.search-result-crumbs-wrap){max-width:30%;margin-left:auto;margin-top:.5em;margin-bottom:.1rem}.aa-DetachedOverlay .aa-Item .search-item .search-result-crumbs.search-result-crumbs-wrap,#quarto-search-results .aa-Item .search-item .search-result-crumbs.search-result-crumbs-wrap{flex-basis:100%;margin-top:0em;margin-bottom:.2em;margin-left:37px}.aa-DetachedOverlay .aa-Item .search-result-title-container,#quarto-search-results .aa-Item .search-result-title-container{font-size:1em;display:flex;flex-wrap:wrap;padding:6px 4px 6px 4px}.aa-DetachedOverlay .aa-Item .search-result-text-container,#quarto-search-results .aa-Item .search-result-text-container{padding-bottom:8px;padding-right:8px;margin-left:42px}.aa-DetachedOverlay .aa-Item .search-result-doc-section,.aa-DetachedOverlay .aa-Item .search-result-more,#quarto-search-results .aa-Item .search-result-doc-section,#quarto-search-results .aa-Item .search-result-more{padding-top:8px;padding-bottom:8px;padding-left:44px}.aa-DetachedOverlay .aa-Item .search-result-more,#quarto-search-results .aa-Item .search-result-more{font-size:.8em;font-weight:400}.aa-DetachedOverlay .aa-Item .search-result-doc,#quarto-search-results .aa-Item .search-result-doc{border-top:1px solid #dee2e6}.aa-DetachedSearchButton{background:none;border:none}.aa-DetachedSearchButton .aa-DetachedSearchButtonPlaceholder{display:none}.navbar .aa-DetachedSearchButton .aa-DetachedSearchButtonIcon{color:#fefdfd}.sidebar-tools-collapse #quarto-search,.sidebar-tools-main #quarto-search{display:inline}.sidebar-tools-collapse #quarto-search .aa-Autocomplete,.sidebar-tools-main #quarto-search .aa-Autocomplete{display:inline}.sidebar-tools-collapse #quarto-search .aa-DetachedSearchButton,.sidebar-tools-main #quarto-search .aa-DetachedSearchButton{padding-left:4px;padding-right:4px}.sidebar-tools-collapse #quarto-search .aa-DetachedSearchButton .aa-DetachedSearchButtonIcon,.sidebar-tools-main #quarto-search .aa-DetachedSearchButton .aa-DetachedSearchButtonIcon{color:#595959}.sidebar-tools-collapse #quarto-search .aa-DetachedSearchButton .aa-DetachedSearchButtonIcon .aa-SubmitIcon,.sidebar-tools-main #quarto-search .aa-DetachedSearchButton .aa-DetachedSearchButtonIcon .aa-SubmitIcon{margin-top:-3px}.aa-DetachedContainer{background:rgba(255,255,255,.65);width:90%;bottom:0;box-shadow:rgba(222,226,230,.6) 0 0 0 1px;outline:currentColor none medium;display:flex;flex-direction:column;left:0;margin:0;overflow:hidden;padding:0;position:fixed;right:0;top:0;z-index:1101}.aa-DetachedContainer::after{height:32px}.aa-DetachedContainer .aa-SourceHeader{margin:var(--aa-spacing-half) 0 var(--aa-spacing-half) 2px}.aa-DetachedContainer .aa-Panel{background-color:#fff;border-radius:0;box-shadow:none;flex-grow:1;margin:0;padding:0;position:relative}.aa-DetachedContainer .aa-PanelLayout{bottom:0;box-shadow:none;left:0;margin:0;max-height:none;overflow-y:auto;position:absolute;right:0;top:0;width:100%}.aa-DetachedFormContainer{background-color:#fff;border-bottom:1px solid #dee2e6;display:flex;flex-direction:row;justify-content:space-between;margin:0;padding:.5em}.aa-DetachedCancelButton{background:none;font-size:.8em;border:0;border-radius:3px;color:#343a40;cursor:pointer;margin:0 0 0 .5em;padding:0 .5em}.aa-DetachedCancelButton:hover,.aa-DetachedCancelButton:focus{box-shadow:rgba(39,128,227,.6) 0 0 0 1px;outline:currentColor none medium}.aa-DetachedContainer--modal{bottom:inherit;height:auto;margin:0 auto;position:absolute;top:100px;border-radius:6px;max-width:850px}@media(max-width: 575.98px){.aa-DetachedContainer--modal{width:100%;top:0px;border-radius:0px;border:none}}.aa-DetachedContainer--modal .aa-PanelLayout{max-height:var(--aa-detached-modal-max-height);padding-bottom:var(--aa-spacing-half);position:static}.aa-Detached{height:100vh;overflow:hidden}.aa-DetachedOverlay{background-color:rgba(52,58,64,.4);position:fixed;left:0;right:0;top:0;margin:0;padding:0;height:100vh;z-index:1100}.quarto-dashboard.nav-fixed.dashboard-sidebar #quarto-content.quarto-dashboard-content{padding:0em}.quarto-dashboard #quarto-content.quarto-dashboard-content{padding:1em}.quarto-dashboard #quarto-content.quarto-dashboard-content>*{padding-top:0}@media(min-width: 576px){.quarto-dashboard{height:100%}}.quarto-dashboard .card.valuebox.bslib-card.bg-primary{background-color:#5397e9 !important}.quarto-dashboard .card.valuebox.bslib-card.bg-secondary{background-color:#343a40 !important}.quarto-dashboard .card.valuebox.bslib-card.bg-success{background-color:#3aa716 !important}.quarto-dashboard .card.valuebox.bslib-card.bg-info{background-color:rgba(153,84,187,.7019607843) !important}.quarto-dashboard .card.valuebox.bslib-card.bg-warning{background-color:#fa6400 !important}.quarto-dashboard .card.valuebox.bslib-card.bg-danger{background-color:rgba(255,0,57,.7019607843) !important}.quarto-dashboard .card.valuebox.bslib-card.bg-light{background-color:#f8f9fa !important}.quarto-dashboard .card.valuebox.bslib-card.bg-dark{background-color:#343a40 !important}.quarto-dashboard.dashboard-fill{display:flex;flex-direction:column}.quarto-dashboard #quarto-appendix{display:none}.quarto-dashboard #quarto-header #quarto-dashboard-header{border-top:solid 1px #df4079;border-bottom:solid 1px #df4079}.quarto-dashboard #quarto-header #quarto-dashboard-header>nav{padding-left:1em;padding-right:1em}.quarto-dashboard #quarto-header #quarto-dashboard-header>nav .navbar-brand-container{padding-left:0}.quarto-dashboard #quarto-header #quarto-dashboard-header .navbar-toggler{margin-right:0}.quarto-dashboard #quarto-header #quarto-dashboard-header .navbar-toggler-icon{height:1em;width:1em;background-image:url('data:image/svg+xml,')}.quarto-dashboard #quarto-header #quarto-dashboard-header .navbar-brand-container{padding-right:1em}.quarto-dashboard #quarto-header #quarto-dashboard-header .navbar-title{font-size:1.1em}.quarto-dashboard #quarto-header #quarto-dashboard-header .navbar-nav{font-size:.9em}.quarto-dashboard #quarto-dashboard-header .navbar{padding:0}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-container{padding-left:1em}.quarto-dashboard #quarto-dashboard-header .navbar.slim .navbar-brand-container .nav-link,.quarto-dashboard #quarto-dashboard-header .navbar.slim .navbar-nav .nav-link{padding:.7em}.quarto-dashboard #quarto-dashboard-header .navbar .quarto-color-scheme-toggle{order:9}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-toggler{margin-left:.5em;order:10}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-nav .nav-link{padding:.5em;height:100%;display:flex;align-items:center}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-nav .active{background-color:#dd3773}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-brand-container{padding:.5em .5em .5em 0;display:flex;flex-direction:row;margin-right:2em;align-items:center}@media(max-width: 767.98px){.quarto-dashboard #quarto-dashboard-header .navbar .navbar-brand-container{margin-right:auto}}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-collapse{align-self:stretch}@media(min-width: 768px){.quarto-dashboard #quarto-dashboard-header .navbar .navbar-collapse{order:8}}@media(max-width: 767.98px){.quarto-dashboard #quarto-dashboard-header .navbar .navbar-collapse{order:1000;padding-bottom:.5em}}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-collapse .navbar-nav{align-self:stretch}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-title{font-size:1.25em;line-height:1.1em;display:flex;flex-direction:row;flex-wrap:wrap;align-items:baseline}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-title .navbar-title-text{margin-right:.4em}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-title a{text-decoration:none;color:inherit}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-subtitle,.quarto-dashboard #quarto-dashboard-header .navbar .navbar-author{font-size:.9rem;margin-right:.5em}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-author{margin-left:auto}.quarto-dashboard #quarto-dashboard-header .navbar .navbar-logo{max-height:48px;min-height:30px;object-fit:cover;margin-right:1em}.quarto-dashboard #quarto-dashboard-header .navbar .quarto-dashboard-links{order:9;padding-right:1em}.quarto-dashboard #quarto-dashboard-header .navbar .quarto-dashboard-link-text{margin-left:.25em}.quarto-dashboard #quarto-dashboard-header .navbar .quarto-dashboard-link{padding-right:0em;padding-left:.7em;text-decoration:none;color:#fefdfd}.quarto-dashboard .page-layout-custom .tab-content{padding:0;border:none}.quarto-dashboard-img-contain{height:100%;width:100%;object-fit:contain}@media(max-width: 575.98px){.quarto-dashboard .bslib-grid{grid-template-rows:minmax(1em, max-content) !important}.quarto-dashboard .sidebar-content{height:inherit}.quarto-dashboard .page-layout-custom{min-height:100vh}}.quarto-dashboard.dashboard-toolbar>.page-layout-custom,.quarto-dashboard.dashboard-sidebar>.page-layout-custom{padding:0}.quarto-dashboard .quarto-dashboard-content.quarto-dashboard-pages{padding:0}.quarto-dashboard .callout{margin-bottom:0;margin-top:0}.quarto-dashboard .html-fill-container figure{overflow:hidden}.quarto-dashboard bslib-tooltip .rounded-pill{border:solid #6c757d 1px}.quarto-dashboard bslib-tooltip .rounded-pill .svg{fill:#343a40}.quarto-dashboard .tabset .dashboard-card-no-title .nav-tabs{margin-left:0;margin-right:auto}.quarto-dashboard .tabset .tab-content{border:none}.quarto-dashboard .tabset .card-header .nav-link[role=tab]{margin-top:-6px;padding-top:6px;padding-bottom:6px}.quarto-dashboard .card.valuebox,.quarto-dashboard .card.bslib-value-box{min-height:3rem}.quarto-dashboard .card.valuebox .card-body,.quarto-dashboard .card.bslib-value-box .card-body{padding:0}.quarto-dashboard .bslib-value-box .value-box-value{font-size:clamp(.1em,15cqw,5em)}.quarto-dashboard .bslib-value-box .value-box-showcase .bi{font-size:clamp(.1em,max(18cqw,5.2cqh),5em);text-align:center;height:1em}.quarto-dashboard .bslib-value-box .value-box-showcase .bi::before{vertical-align:1em}.quarto-dashboard .bslib-value-box .value-box-area{margin-top:auto;margin-bottom:auto}.quarto-dashboard .card figure.quarto-float{display:flex;flex-direction:column;align-items:center}.quarto-dashboard .dashboard-scrolling{padding:1em}.quarto-dashboard .full-height{height:100%}.quarto-dashboard .showcase-bottom .value-box-grid{display:grid;grid-template-columns:1fr;grid-template-rows:1fr auto;grid-template-areas:"top" "bottom"}.quarto-dashboard .showcase-bottom .value-box-grid .value-box-showcase{grid-area:bottom;padding:0;margin:0}.quarto-dashboard .showcase-bottom .value-box-grid .value-box-showcase i.bi{font-size:4rem}.quarto-dashboard .showcase-bottom .value-box-grid .value-box-area{grid-area:top}.quarto-dashboard .tab-content{margin-bottom:0}.quarto-dashboard .bslib-card .bslib-navs-card-title{justify-content:stretch;align-items:end}.quarto-dashboard .card-header{display:flex;flex-wrap:wrap;justify-content:space-between}.quarto-dashboard .card-header .card-title{display:flex;flex-direction:column;justify-content:center;margin-bottom:0}.quarto-dashboard .tabset .card-toolbar{margin-bottom:1em}.quarto-dashboard .bslib-grid>.bslib-sidebar-layout{border:none;gap:var(--bslib-spacer, 1rem)}.quarto-dashboard .bslib-grid>.bslib-sidebar-layout>.main{padding:0}.quarto-dashboard .bslib-grid>.bslib-sidebar-layout>.sidebar{border-radius:.25rem;border:1px solid rgba(0,0,0,.175)}.quarto-dashboard .bslib-grid>.bslib-sidebar-layout>.collapse-toggle{display:none}@media(max-width: 767.98px){.quarto-dashboard .bslib-grid>.bslib-sidebar-layout{grid-template-columns:1fr;grid-template-rows:max-content 1fr}.quarto-dashboard .bslib-grid>.bslib-sidebar-layout>.main{grid-column:1;grid-row:2}.quarto-dashboard .bslib-grid>.bslib-sidebar-layout .sidebar{grid-column:1;grid-row:1}}.quarto-dashboard .sidebar-right .sidebar{padding-left:2.5em}.quarto-dashboard .sidebar-right .collapse-toggle{left:2px}.quarto-dashboard .quarto-dashboard .sidebar-right button.collapse-toggle:not(.transitioning){left:unset}.quarto-dashboard aside.sidebar{padding-left:1em;padding-right:1em;background-color:rgba(52,58,64,.25);color:#343a40}.quarto-dashboard .bslib-sidebar-layout>div.main{padding:.7em}.quarto-dashboard .bslib-sidebar-layout button.collapse-toggle{margin-top:.3em}.quarto-dashboard .bslib-sidebar-layout .collapse-toggle{top:0}.quarto-dashboard .bslib-sidebar-layout.sidebar-collapsed:not(.transitioning):not(.sidebar-right) .collapse-toggle{left:2px}.quarto-dashboard .sidebar>section>.h3:first-of-type{margin-top:0em}.quarto-dashboard .sidebar .h3,.quarto-dashboard .sidebar .h4,.quarto-dashboard .sidebar .h5,.quarto-dashboard .sidebar .h6{margin-top:.5em}.quarto-dashboard .sidebar form{flex-direction:column;align-items:start;margin-bottom:1em}.quarto-dashboard .sidebar form div[class*=oi-][class$=-input]{flex-direction:column}.quarto-dashboard .sidebar form[class*=oi-][class$=-toggle]{flex-direction:row-reverse;align-items:center;justify-content:start}.quarto-dashboard .sidebar form input[type=range]{margin-top:.5em;margin-right:.8em;margin-left:1em}.quarto-dashboard .sidebar label{width:fit-content}.quarto-dashboard .sidebar .card-body{margin-bottom:2em}.quarto-dashboard .sidebar .shiny-input-container{margin-bottom:1em}.quarto-dashboard .sidebar .shiny-options-group{margin-top:0}.quarto-dashboard .sidebar .control-label{margin-bottom:.3em}.quarto-dashboard .card .card-body .quarto-layout-row{align-items:stretch}.quarto-dashboard .toolbar{font-size:.9em;display:flex;flex-direction:row;border-top:solid 1px #bcbfc0;padding:1em;flex-wrap:wrap;background-color:rgba(52,58,64,.25)}.quarto-dashboard .toolbar .cell-output-display{display:flex}.quarto-dashboard .toolbar .shiny-input-container{padding-bottom:.5em;margin-bottom:.5em;width:inherit}.quarto-dashboard .toolbar .shiny-input-container>.checkbox:first-child{margin-top:6px}.quarto-dashboard .toolbar>*:last-child{margin-right:0}.quarto-dashboard .toolbar>*>*{margin-right:1em;align-items:baseline}.quarto-dashboard .toolbar>*>*>a{text-decoration:none;margin-top:auto;margin-bottom:auto}.quarto-dashboard .toolbar .shiny-input-container{padding-bottom:0;margin-bottom:0}.quarto-dashboard .toolbar .shiny-input-container>*{flex-shrink:0;flex-grow:0}.quarto-dashboard .toolbar .form-group.shiny-input-container:not([role=group])>label{margin-bottom:0}.quarto-dashboard .toolbar .shiny-input-container.no-baseline{align-items:start;padding-top:6px}.quarto-dashboard .toolbar .shiny-input-container{display:flex;align-items:baseline}.quarto-dashboard .toolbar .shiny-input-container label{padding-right:.4em}.quarto-dashboard .toolbar .shiny-input-container .bslib-input-switch{margin-top:6px}.quarto-dashboard .toolbar input[type=text]{line-height:1;width:inherit}.quarto-dashboard .toolbar .input-daterange{width:inherit}.quarto-dashboard .toolbar .input-daterange input[type=text]{height:2.4em;width:10em}.quarto-dashboard .toolbar .input-daterange .input-group-addon{height:auto;padding:0;margin-left:-5px !important;margin-right:-5px}.quarto-dashboard .toolbar .input-daterange .input-group-addon .input-group-text{padding-top:0;padding-bottom:0;height:100%}.quarto-dashboard .toolbar span.irs.irs--shiny{width:10em}.quarto-dashboard .toolbar span.irs.irs--shiny .irs-line{top:9px}.quarto-dashboard .toolbar span.irs.irs--shiny .irs-min,.quarto-dashboard .toolbar span.irs.irs--shiny .irs-max,.quarto-dashboard .toolbar span.irs.irs--shiny .irs-from,.quarto-dashboard .toolbar span.irs.irs--shiny .irs-to,.quarto-dashboard .toolbar span.irs.irs--shiny .irs-single{top:20px}.quarto-dashboard .toolbar span.irs.irs--shiny .irs-bar{top:8px}.quarto-dashboard .toolbar span.irs.irs--shiny .irs-handle{top:0px}.quarto-dashboard .toolbar .shiny-input-checkboxgroup>label{margin-top:6px}.quarto-dashboard .toolbar .shiny-input-checkboxgroup>.shiny-options-group{margin-top:0;align-items:baseline}.quarto-dashboard .toolbar .shiny-input-radiogroup>label{margin-top:6px}.quarto-dashboard .toolbar .shiny-input-radiogroup>.shiny-options-group{align-items:baseline;margin-top:0}.quarto-dashboard .toolbar .shiny-input-radiogroup>.shiny-options-group>.radio{margin-right:.3em}.quarto-dashboard .toolbar .form-select{padding-top:.2em;padding-bottom:.2em}.quarto-dashboard .toolbar .shiny-input-select{min-width:6em}.quarto-dashboard .toolbar div.checkbox{margin-bottom:0px}.quarto-dashboard .toolbar>.checkbox:first-child{margin-top:6px}.quarto-dashboard .toolbar form{width:fit-content}.quarto-dashboard .toolbar form label{padding-top:.2em;padding-bottom:.2em;width:fit-content}.quarto-dashboard .toolbar form input[type=date]{width:fit-content}.quarto-dashboard .toolbar form input[type=color]{width:3em}.quarto-dashboard .toolbar form button{padding:.4em}.quarto-dashboard .toolbar form select{width:fit-content}.quarto-dashboard .toolbar>*{font-size:.9em;flex-grow:0}.quarto-dashboard .toolbar .shiny-input-container label{margin-bottom:1px}.quarto-dashboard .toolbar-bottom{margin-top:1em;margin-bottom:0 !important;order:2}.quarto-dashboard .quarto-dashboard-content>.dashboard-toolbar-container>.toolbar-content>.tab-content>.tab-pane>*:not(.bslib-sidebar-layout){padding:1em}.quarto-dashboard .quarto-dashboard-content>.dashboard-toolbar-container>.toolbar-content>*:not(.tab-content){padding:1em}.quarto-dashboard .quarto-dashboard-content>.tab-content>.dashboard-page>.dashboard-toolbar-container>.toolbar-content,.quarto-dashboard .quarto-dashboard-content>.tab-content>.dashboard-page:not(.dashboard-sidebar-container)>*:not(.dashboard-toolbar-container){padding:1em}.quarto-dashboard .toolbar-content{padding:0}.quarto-dashboard .quarto-dashboard-content.quarto-dashboard-pages .tab-pane>.dashboard-toolbar-container .toolbar{border-radius:0;margin-bottom:0}.quarto-dashboard .dashboard-toolbar-container.toolbar-toplevel .toolbar{border-bottom:1px solid rgba(0,0,0,.175)}.quarto-dashboard .dashboard-toolbar-container.toolbar-toplevel .toolbar-bottom{margin-top:0}.quarto-dashboard .dashboard-toolbar-container:not(.toolbar-toplevel) .toolbar{margin-bottom:1em;border-top:none;border-radius:.25rem;border:1px solid rgba(0,0,0,.175)}.quarto-dashboard .vega-embed.has-actions details{width:1.7em;height:2em;position:absolute !important;top:0;right:0}.quarto-dashboard .dashboard-toolbar-container{padding:0}.quarto-dashboard .card .card-header p:last-child,.quarto-dashboard .card .card-footer p:last-child{margin-bottom:0}.quarto-dashboard .card .card-body>.h4:first-child{margin-top:0}.quarto-dashboard .card .card-body{z-index:4}@media(max-width: 767.98px){.quarto-dashboard .card .card-body .itables div.dataTables_wrapper div.dataTables_length,.quarto-dashboard .card .card-body .itables div.dataTables_wrapper div.dataTables_info,.quarto-dashboard .card .card-body .itables div.dataTables_wrapper div.dataTables_paginate{text-align:initial}.quarto-dashboard .card .card-body .itables div.dataTables_wrapper div.dataTables_filter{text-align:right}.quarto-dashboard .card .card-body .itables div.dataTables_wrapper div.dataTables_paginate ul.pagination{justify-content:initial}}.quarto-dashboard .card .card-body .itables .dataTables_wrapper{display:flex;flex-wrap:wrap;justify-content:space-between;align-items:center;padding-top:0}.quarto-dashboard .card .card-body .itables .dataTables_wrapper table{flex-shrink:0}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dt-buttons{margin-bottom:.5em;margin-left:auto;width:fit-content;float:right}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dt-buttons.btn-group{background:#fff;border:none}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dt-buttons .btn-secondary{background-color:#fff;background-image:none;border:solid #dee2e6 1px;padding:.2em .7em}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dt-buttons .btn span{font-size:.8em;color:#343a40}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_info{margin-left:.5em;margin-bottom:.5em;padding-top:0}@media(min-width: 768px){.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_info{font-size:.875em}}@media(max-width: 767.98px){.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_info{font-size:.8em}}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_filter{margin-bottom:.5em;font-size:.875em}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_filter input[type=search]{padding:1px 5px 1px 5px;font-size:.875em}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_length{flex-basis:1 1 50%;margin-bottom:.5em;font-size:.875em}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_length select{padding:.4em 3em .4em .5em;font-size:.875em;margin-left:.2em;margin-right:.2em}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_paginate{flex-shrink:0}@media(min-width: 768px){.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_paginate{margin-left:auto}}.quarto-dashboard .card .card-body .itables .dataTables_wrapper .dataTables_paginate ul.pagination .paginate_button .page-link{font-size:.8em}.quarto-dashboard .card .card-footer{font-size:.9em}.quarto-dashboard .card .card-toolbar{display:flex;flex-grow:1;flex-direction:row;width:100%;flex-wrap:wrap}.quarto-dashboard .card .card-toolbar>*{font-size:.8em;flex-grow:0}.quarto-dashboard .card .card-toolbar>.card-title{font-size:1em;flex-grow:1;align-self:flex-start;margin-top:.1em}.quarto-dashboard .card .card-toolbar .cell-output-display{display:flex}.quarto-dashboard .card .card-toolbar .shiny-input-container{padding-bottom:.5em;margin-bottom:.5em;width:inherit}.quarto-dashboard .card .card-toolbar .shiny-input-container>.checkbox:first-child{margin-top:6px}.quarto-dashboard .card .card-toolbar>*:last-child{margin-right:0}.quarto-dashboard .card .card-toolbar>*>*{margin-right:1em;align-items:baseline}.quarto-dashboard .card .card-toolbar>*>*>a{text-decoration:none;margin-top:auto;margin-bottom:auto}.quarto-dashboard .card .card-toolbar form{width:fit-content}.quarto-dashboard .card .card-toolbar form label{padding-top:.2em;padding-bottom:.2em;width:fit-content}.quarto-dashboard .card .card-toolbar form input[type=date]{width:fit-content}.quarto-dashboard .card .card-toolbar form input[type=color]{width:3em}.quarto-dashboard .card .card-toolbar form button{padding:.4em}.quarto-dashboard .card .card-toolbar form select{width:fit-content}.quarto-dashboard .card .card-toolbar .cell-output-display{display:flex}.quarto-dashboard .card .card-toolbar .shiny-input-container{padding-bottom:.5em;margin-bottom:.5em;width:inherit}.quarto-dashboard .card .card-toolbar .shiny-input-container>.checkbox:first-child{margin-top:6px}.quarto-dashboard .card .card-toolbar>*:last-child{margin-right:0}.quarto-dashboard .card .card-toolbar>*>*{margin-right:1em;align-items:baseline}.quarto-dashboard .card .card-toolbar>*>*>a{text-decoration:none;margin-top:auto;margin-bottom:auto}.quarto-dashboard .card .card-toolbar .shiny-input-container{padding-bottom:0;margin-bottom:0}.quarto-dashboard .card .card-toolbar .shiny-input-container>*{flex-shrink:0;flex-grow:0}.quarto-dashboard .card .card-toolbar .form-group.shiny-input-container:not([role=group])>label{margin-bottom:0}.quarto-dashboard .card .card-toolbar .shiny-input-container.no-baseline{align-items:start;padding-top:6px}.quarto-dashboard .card .card-toolbar .shiny-input-container{display:flex;align-items:baseline}.quarto-dashboard .card .card-toolbar .shiny-input-container label{padding-right:.4em}.quarto-dashboard .card .card-toolbar .shiny-input-container .bslib-input-switch{margin-top:6px}.quarto-dashboard .card .card-toolbar input[type=text]{line-height:1;width:inherit}.quarto-dashboard .card .card-toolbar .input-daterange{width:inherit}.quarto-dashboard .card .card-toolbar .input-daterange input[type=text]{height:2.4em;width:10em}.quarto-dashboard .card .card-toolbar .input-daterange .input-group-addon{height:auto;padding:0;margin-left:-5px !important;margin-right:-5px}.quarto-dashboard .card .card-toolbar .input-daterange .input-group-addon .input-group-text{padding-top:0;padding-bottom:0;height:100%}.quarto-dashboard .card .card-toolbar span.irs.irs--shiny{width:10em}.quarto-dashboard .card .card-toolbar span.irs.irs--shiny .irs-line{top:9px}.quarto-dashboard .card .card-toolbar span.irs.irs--shiny .irs-min,.quarto-dashboard .card .card-toolbar span.irs.irs--shiny .irs-max,.quarto-dashboard .card .card-toolbar span.irs.irs--shiny .irs-from,.quarto-dashboard .card .card-toolbar span.irs.irs--shiny .irs-to,.quarto-dashboard .card .card-toolbar span.irs.irs--shiny .irs-single{top:20px}.quarto-dashboard .card .card-toolbar span.irs.irs--shiny .irs-bar{top:8px}.quarto-dashboard .card .card-toolbar span.irs.irs--shiny .irs-handle{top:0px}.quarto-dashboard .card .card-toolbar .shiny-input-checkboxgroup>label{margin-top:6px}.quarto-dashboard .card .card-toolbar .shiny-input-checkboxgroup>.shiny-options-group{margin-top:0;align-items:baseline}.quarto-dashboard .card .card-toolbar .shiny-input-radiogroup>label{margin-top:6px}.quarto-dashboard .card .card-toolbar .shiny-input-radiogroup>.shiny-options-group{align-items:baseline;margin-top:0}.quarto-dashboard .card .card-toolbar .shiny-input-radiogroup>.shiny-options-group>.radio{margin-right:.3em}.quarto-dashboard .card .card-toolbar .form-select{padding-top:.2em;padding-bottom:.2em}.quarto-dashboard .card .card-toolbar .shiny-input-select{min-width:6em}.quarto-dashboard .card .card-toolbar div.checkbox{margin-bottom:0px}.quarto-dashboard .card .card-toolbar>.checkbox:first-child{margin-top:6px}.quarto-dashboard .card-body>table>thead{border-top:none}.quarto-dashboard .card-body>.table>:not(caption)>*>*{background-color:#fff}.tableFloatingHeaderOriginal{background-color:#fff;position:sticky !important;top:0 !important}.dashboard-data-table{margin-top:-1px}div.value-box-area span.observablehq--number{font-size:calc(clamp(.1em,15cqw,5em)*1.25);line-height:1.2;color:inherit;font-family:var(--bs-body-font-family)}.quarto-listing{padding-bottom:1em}.listing-pagination{padding-top:.5em}ul.pagination{float:right;padding-left:8px;padding-top:.5em}ul.pagination li{padding-right:.75em}ul.pagination li.disabled a,ul.pagination li.active a{color:#fff;text-decoration:none}ul.pagination li:last-of-type{padding-right:0}.listing-actions-group{display:flex}.quarto-listing-filter{margin-bottom:1em;width:200px;margin-left:auto}.quarto-listing-sort{margin-bottom:1em;margin-right:auto;width:auto}.quarto-listing-sort .input-group-text{font-size:.8em}.input-group-text{border-right:none}.quarto-listing-sort select.form-select{font-size:.8em}.listing-no-matching{text-align:center;padding-top:2em;padding-bottom:3em;font-size:1em}#quarto-margin-sidebar .quarto-listing-category{padding-top:0;font-size:1rem}#quarto-margin-sidebar .quarto-listing-category-title{cursor:pointer;font-weight:600;font-size:1rem}.quarto-listing-category .category{cursor:pointer}.quarto-listing-category .category.active{font-weight:600}.quarto-listing-category.category-cloud{display:flex;flex-wrap:wrap;align-items:baseline}.quarto-listing-category.category-cloud .category{padding-right:5px}.quarto-listing-category.category-cloud .category-cloud-1{font-size:.75em}.quarto-listing-category.category-cloud .category-cloud-2{font-size:.95em}.quarto-listing-category.category-cloud .category-cloud-3{font-size:1.15em}.quarto-listing-category.category-cloud .category-cloud-4{font-size:1.35em}.quarto-listing-category.category-cloud .category-cloud-5{font-size:1.55em}.quarto-listing-category.category-cloud .category-cloud-6{font-size:1.75em}.quarto-listing-category.category-cloud .category-cloud-7{font-size:1.95em}.quarto-listing-category.category-cloud .category-cloud-8{font-size:2.15em}.quarto-listing-category.category-cloud .category-cloud-9{font-size:2.35em}.quarto-listing-category.category-cloud .category-cloud-10{font-size:2.55em}.quarto-listing-cols-1{grid-template-columns:repeat(1, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-1{grid-template-columns:repeat(1, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-1{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-2{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-2{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-2{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-3{grid-template-columns:repeat(3, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-3{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-3{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-4{grid-template-columns:repeat(4, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-4{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-4{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-5{grid-template-columns:repeat(5, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-5{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-5{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-6{grid-template-columns:repeat(6, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-6{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-6{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-7{grid-template-columns:repeat(7, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-7{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-7{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-8{grid-template-columns:repeat(8, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-8{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-8{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-9{grid-template-columns:repeat(9, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-9{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-9{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-10{grid-template-columns:repeat(10, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-10{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-10{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-11{grid-template-columns:repeat(11, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-11{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-11{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-cols-12{grid-template-columns:repeat(12, minmax(0, 1fr));gap:1.5em}@media(max-width: 767.98px){.quarto-listing-cols-12{grid-template-columns:repeat(2, minmax(0, 1fr));gap:1.5em}}@media(max-width: 575.98px){.quarto-listing-cols-12{grid-template-columns:minmax(0, 1fr);gap:1.5em}}.quarto-listing-grid{gap:1.5em}.quarto-grid-item.borderless{border:none}.quarto-grid-item.borderless .listing-categories .listing-category:last-of-type,.quarto-grid-item.borderless .listing-categories .listing-category:first-of-type{padding-left:0}.quarto-grid-item.borderless .listing-categories .listing-category{border:0}.quarto-grid-link{text-decoration:none;color:inherit}.quarto-grid-link:hover{text-decoration:none;color:inherit}.quarto-grid-item h5.title,.quarto-grid-item .title.h5{margin-top:0;margin-bottom:0}.quarto-grid-item .card-footer{display:flex;justify-content:space-between;font-size:.8em}.quarto-grid-item .card-footer p{margin-bottom:0}.quarto-grid-item p.card-img-top{margin-bottom:0}.quarto-grid-item p.card-img-top>img{object-fit:cover}.quarto-grid-item .card-other-values{margin-top:.5em;font-size:.8em}.quarto-grid-item .card-other-values tr{margin-bottom:.5em}.quarto-grid-item .card-other-values tr>td:first-of-type{font-weight:600;padding-right:1em;padding-left:1em;vertical-align:top}.quarto-grid-item div.post-contents{display:flex;flex-direction:column;text-decoration:none;height:100%}.quarto-grid-item .listing-item-img-placeholder{background-color:rgba(52,58,64,.25);flex-shrink:0}.quarto-grid-item .card-attribution{padding-top:1em;display:flex;gap:1em;text-transform:uppercase;color:#6c757d;font-weight:500;flex-grow:10;align-items:flex-end}.quarto-grid-item .description{padding-bottom:1em}.quarto-grid-item .card-attribution .date{align-self:flex-end}.quarto-grid-item .card-attribution.justify{justify-content:space-between}.quarto-grid-item .card-attribution.start{justify-content:flex-start}.quarto-grid-item .card-attribution.end{justify-content:flex-end}.quarto-grid-item .card-title{margin-bottom:.1em}.quarto-grid-item .card-subtitle{padding-top:.25em}.quarto-grid-item .card-text{font-size:.9em}.quarto-grid-item .listing-reading-time{padding-bottom:.25em}.quarto-grid-item .card-text-small{font-size:.8em}.quarto-grid-item .card-subtitle.subtitle{font-size:.9em;font-weight:600;padding-bottom:.5em}.quarto-grid-item .listing-categories{display:flex;flex-wrap:wrap;padding-bottom:5px}.quarto-grid-item .listing-categories .listing-category{color:#6c757d;border:solid 1px #dee2e6;border-radius:.25rem;text-transform:uppercase;font-size:.65em;padding-left:.5em;padding-right:.5em;padding-top:.15em;padding-bottom:.15em;cursor:pointer;margin-right:4px;margin-bottom:4px}.quarto-grid-item.card-right{text-align:right}.quarto-grid-item.card-right .listing-categories{justify-content:flex-end}.quarto-grid-item.card-left{text-align:left}.quarto-grid-item.card-center{text-align:center}.quarto-grid-item.card-center .listing-description{text-align:justify}.quarto-grid-item.card-center .listing-categories{justify-content:center}table.quarto-listing-table td.image{padding:0px}table.quarto-listing-table td.image img{width:100%;max-width:50px;object-fit:contain}table.quarto-listing-table a{text-decoration:none;word-break:keep-all}table.quarto-listing-table th a{color:inherit}table.quarto-listing-table th a.asc:after{margin-bottom:-2px;margin-left:5px;display:inline-block;height:1rem;width:1rem;background-repeat:no-repeat;background-size:1rem 1rem;background-image:url('data:image/svg+xml,');content:""}table.quarto-listing-table th a.desc:after{margin-bottom:-2px;margin-left:5px;display:inline-block;height:1rem;width:1rem;background-repeat:no-repeat;background-size:1rem 1rem;background-image:url('data:image/svg+xml,');content:""}table.quarto-listing-table.table-hover td{cursor:pointer}.quarto-post.image-left{flex-direction:row}.quarto-post.image-right{flex-direction:row-reverse}@media(max-width: 767.98px){.quarto-post.image-right,.quarto-post.image-left{gap:0em;flex-direction:column}.quarto-post .metadata{padding-bottom:1em;order:2}.quarto-post .body{order:1}.quarto-post .thumbnail{order:3}}.list.quarto-listing-default div:last-of-type{border-bottom:none}@media(min-width: 992px){.quarto-listing-container-default{margin-right:2em}}div.quarto-post{display:flex;gap:2em;margin-bottom:1.5em;border-bottom:1px solid #dee2e6}@media(max-width: 767.98px){div.quarto-post{padding-bottom:1em}}div.quarto-post .metadata{flex-basis:20%;flex-grow:0;margin-top:.2em;flex-shrink:10}div.quarto-post .thumbnail{flex-basis:30%;flex-grow:0;flex-shrink:0}div.quarto-post .thumbnail img{margin-top:.4em;width:100%;object-fit:cover}div.quarto-post .body{flex-basis:45%;flex-grow:1;flex-shrink:0}div.quarto-post .body h3.listing-title,div.quarto-post .body .listing-title.h3{margin-top:0px;margin-bottom:0px;border-bottom:none}div.quarto-post .body .listing-subtitle{font-size:.875em;margin-bottom:.5em;margin-top:.2em}div.quarto-post .body .description{font-size:.9em}div.quarto-post .body pre code{white-space:pre-wrap}div.quarto-post a{color:#343a40;text-decoration:none}div.quarto-post .metadata{display:flex;flex-direction:column;font-size:.8em;font-family:"Source Sans Pro",-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";flex-basis:33%}div.quarto-post .listing-categories{display:flex;flex-wrap:wrap;padding-bottom:5px}div.quarto-post .listing-categories .listing-category{color:#6c757d;border:solid 1px #dee2e6;border-radius:.25rem;text-transform:uppercase;font-size:.65em;padding-left:.5em;padding-right:.5em;padding-top:.15em;padding-bottom:.15em;cursor:pointer;margin-right:4px;margin-bottom:4px}div.quarto-post .listing-description{margin-bottom:.5em}div.quarto-about-jolla{display:flex !important;flex-direction:column;align-items:center;margin-top:10%;padding-bottom:1em}div.quarto-about-jolla .about-image{object-fit:cover;margin-left:auto;margin-right:auto;margin-bottom:1.5em}div.quarto-about-jolla img.round{border-radius:50%}div.quarto-about-jolla img.rounded{border-radius:10px}div.quarto-about-jolla .quarto-title h1.title,div.quarto-about-jolla .quarto-title .title.h1{text-align:center}div.quarto-about-jolla .quarto-title .description{text-align:center}div.quarto-about-jolla h2,div.quarto-about-jolla .h2{border-bottom:none}div.quarto-about-jolla .about-sep{width:60%}div.quarto-about-jolla main{text-align:center}div.quarto-about-jolla .about-links{display:flex}@media(min-width: 992px){div.quarto-about-jolla .about-links{flex-direction:row;column-gap:.8em;row-gap:15px;flex-wrap:wrap}}@media(max-width: 991.98px){div.quarto-about-jolla .about-links{flex-direction:column;row-gap:1em;width:100%;padding-bottom:1.5em}}div.quarto-about-jolla .about-link{color:#626d78;text-decoration:none;border:solid 1px}@media(min-width: 992px){div.quarto-about-jolla .about-link{font-size:.8em;padding:.25em .5em;border-radius:4px}}@media(max-width: 991.98px){div.quarto-about-jolla .about-link{font-size:1.1em;padding:.5em .5em;text-align:center;border-radius:6px}}div.quarto-about-jolla .about-link:hover{color:#2761e3}div.quarto-about-jolla .about-link i.bi{margin-right:.15em}div.quarto-about-solana{display:flex !important;flex-direction:column;padding-top:3em !important;padding-bottom:1em}div.quarto-about-solana .about-entity{display:flex !important;align-items:start;justify-content:space-between}@media(min-width: 992px){div.quarto-about-solana .about-entity{flex-direction:row}}@media(max-width: 991.98px){div.quarto-about-solana .about-entity{flex-direction:column-reverse;align-items:center;text-align:center}}div.quarto-about-solana .about-entity .entity-contents{display:flex;flex-direction:column}@media(max-width: 767.98px){div.quarto-about-solana .about-entity .entity-contents{width:100%}}div.quarto-about-solana .about-entity .about-image{object-fit:cover}@media(max-width: 991.98px){div.quarto-about-solana .about-entity .about-image{margin-bottom:1.5em}}div.quarto-about-solana .about-entity img.round{border-radius:50%}div.quarto-about-solana .about-entity img.rounded{border-radius:10px}div.quarto-about-solana .about-entity .about-links{display:flex;justify-content:left;padding-bottom:1.2em}@media(min-width: 992px){div.quarto-about-solana .about-entity .about-links{flex-direction:row;column-gap:.8em;row-gap:15px;flex-wrap:wrap}}@media(max-width: 991.98px){div.quarto-about-solana .about-entity .about-links{flex-direction:column;row-gap:1em;width:100%;padding-bottom:1.5em}}div.quarto-about-solana .about-entity .about-link{color:#626d78;text-decoration:none;border:solid 1px}@media(min-width: 992px){div.quarto-about-solana .about-entity .about-link{font-size:.8em;padding:.25em .5em;border-radius:4px}}@media(max-width: 991.98px){div.quarto-about-solana .about-entity .about-link{font-size:1.1em;padding:.5em .5em;text-align:center;border-radius:6px}}div.quarto-about-solana .about-entity .about-link:hover{color:#2761e3}div.quarto-about-solana .about-entity .about-link i.bi{margin-right:.15em}div.quarto-about-solana .about-contents{padding-right:1.5em;flex-basis:0;flex-grow:1}div.quarto-about-solana .about-contents main.content{margin-top:0}div.quarto-about-solana .about-contents h2,div.quarto-about-solana .about-contents .h2{border-bottom:none}div.quarto-about-trestles{display:flex !important;flex-direction:row;padding-top:3em !important;padding-bottom:1em}@media(max-width: 991.98px){div.quarto-about-trestles{flex-direction:column;padding-top:0em !important}}div.quarto-about-trestles .about-entity{display:flex !important;flex-direction:column;align-items:center;text-align:center;padding-right:1em}@media(min-width: 992px){div.quarto-about-trestles .about-entity{flex:0 0 42%}}div.quarto-about-trestles .about-entity .about-image{object-fit:cover;margin-bottom:1.5em}div.quarto-about-trestles .about-entity img.round{border-radius:50%}div.quarto-about-trestles .about-entity img.rounded{border-radius:10px}div.quarto-about-trestles .about-entity .about-links{display:flex;justify-content:center}@media(min-width: 992px){div.quarto-about-trestles .about-entity .about-links{flex-direction:row;column-gap:.8em;row-gap:15px;flex-wrap:wrap}}@media(max-width: 991.98px){div.quarto-about-trestles .about-entity .about-links{flex-direction:column;row-gap:1em;width:100%;padding-bottom:1.5em}}div.quarto-about-trestles .about-entity .about-link{color:#626d78;text-decoration:none;border:solid 1px}@media(min-width: 992px){div.quarto-about-trestles .about-entity .about-link{font-size:.8em;padding:.25em .5em;border-radius:4px}}@media(max-width: 991.98px){div.quarto-about-trestles .about-entity .about-link{font-size:1.1em;padding:.5em .5em;text-align:center;border-radius:6px}}div.quarto-about-trestles .about-entity .about-link:hover{color:#2761e3}div.quarto-about-trestles .about-entity .about-link i.bi{margin-right:.15em}div.quarto-about-trestles .about-contents{flex-basis:0;flex-grow:1}div.quarto-about-trestles .about-contents h2,div.quarto-about-trestles .about-contents .h2{border-bottom:none}@media(min-width: 992px){div.quarto-about-trestles .about-contents{border-left:solid 1px #dee2e6;padding-left:1.5em}}div.quarto-about-trestles .about-contents main.content{margin-top:0}div.quarto-about-marquee{padding-bottom:1em}div.quarto-about-marquee .about-contents{display:flex;flex-direction:column}div.quarto-about-marquee .about-image{max-height:550px;margin-bottom:1.5em;object-fit:cover}div.quarto-about-marquee img.round{border-radius:50%}div.quarto-about-marquee img.rounded{border-radius:10px}div.quarto-about-marquee h2,div.quarto-about-marquee .h2{border-bottom:none}div.quarto-about-marquee .about-links{display:flex;justify-content:center;padding-top:1.5em}@media(min-width: 992px){div.quarto-about-marquee .about-links{flex-direction:row;column-gap:.8em;row-gap:15px;flex-wrap:wrap}}@media(max-width: 991.98px){div.quarto-about-marquee .about-links{flex-direction:column;row-gap:1em;width:100%;padding-bottom:1.5em}}div.quarto-about-marquee .about-link{color:#626d78;text-decoration:none;border:solid 1px}@media(min-width: 992px){div.quarto-about-marquee .about-link{font-size:.8em;padding:.25em .5em;border-radius:4px}}@media(max-width: 991.98px){div.quarto-about-marquee .about-link{font-size:1.1em;padding:.5em .5em;text-align:center;border-radius:6px}}div.quarto-about-marquee .about-link:hover{color:#2761e3}div.quarto-about-marquee .about-link i.bi{margin-right:.15em}@media(min-width: 992px){div.quarto-about-marquee .about-link{border:none}}div.quarto-about-broadside{display:flex;flex-direction:column;padding-bottom:1em}div.quarto-about-broadside .about-main{display:flex !important;padding-top:0 !important}@media(min-width: 992px){div.quarto-about-broadside .about-main{flex-direction:row;align-items:flex-start}}@media(max-width: 991.98px){div.quarto-about-broadside .about-main{flex-direction:column}}@media(max-width: 991.98px){div.quarto-about-broadside .about-main .about-entity{flex-shrink:0;width:100%;height:450px;margin-bottom:1.5em;background-size:cover;background-repeat:no-repeat}}@media(min-width: 992px){div.quarto-about-broadside .about-main .about-entity{flex:0 10 50%;margin-right:1.5em;width:100%;height:100%;background-size:100%;background-repeat:no-repeat}}div.quarto-about-broadside .about-main .about-contents{padding-top:14px;flex:0 0 50%}div.quarto-about-broadside h2,div.quarto-about-broadside .h2{border-bottom:none}div.quarto-about-broadside .about-sep{margin-top:1.5em;width:60%;align-self:center}div.quarto-about-broadside .about-links{display:flex;justify-content:center;column-gap:20px;padding-top:1.5em}@media(min-width: 992px){div.quarto-about-broadside .about-links{flex-direction:row;column-gap:.8em;row-gap:15px;flex-wrap:wrap}}@media(max-width: 991.98px){div.quarto-about-broadside .about-links{flex-direction:column;row-gap:1em;width:100%;padding-bottom:1.5em}}div.quarto-about-broadside .about-link{color:#626d78;text-decoration:none;border:solid 1px}@media(min-width: 992px){div.quarto-about-broadside .about-link{font-size:.8em;padding:.25em .5em;border-radius:4px}}@media(max-width: 991.98px){div.quarto-about-broadside .about-link{font-size:1.1em;padding:.5em .5em;text-align:center;border-radius:6px}}div.quarto-about-broadside .about-link:hover{color:#2761e3}div.quarto-about-broadside .about-link i.bi{margin-right:.15em}@media(min-width: 992px){div.quarto-about-broadside .about-link{border:none}}.tippy-box[data-theme~=quarto]{background-color:#fff;border:solid 1px #dee2e6;border-radius:.25rem;color:#343a40;font-size:.875rem}.tippy-box[data-theme~=quarto]>.tippy-backdrop{background-color:#fff}.tippy-box[data-theme~=quarto]>.tippy-arrow:after,.tippy-box[data-theme~=quarto]>.tippy-svg-arrow:after{content:"";position:absolute;z-index:-1}.tippy-box[data-theme~=quarto]>.tippy-arrow:after{border-color:rgba(0,0,0,0);border-style:solid}.tippy-box[data-placement^=top]>.tippy-arrow:before{bottom:-6px}.tippy-box[data-placement^=bottom]>.tippy-arrow:before{top:-6px}.tippy-box[data-placement^=right]>.tippy-arrow:before{left:-6px}.tippy-box[data-placement^=left]>.tippy-arrow:before{right:-6px}.tippy-box[data-theme~=quarto][data-placement^=top]>.tippy-arrow:before{border-top-color:#fff}.tippy-box[data-theme~=quarto][data-placement^=top]>.tippy-arrow:after{border-top-color:#dee2e6;border-width:7px 7px 0;top:17px;left:1px}.tippy-box[data-theme~=quarto][data-placement^=top]>.tippy-svg-arrow>svg{top:16px}.tippy-box[data-theme~=quarto][data-placement^=top]>.tippy-svg-arrow:after{top:17px}.tippy-box[data-theme~=quarto][data-placement^=bottom]>.tippy-arrow:before{border-bottom-color:#fff;bottom:16px}.tippy-box[data-theme~=quarto][data-placement^=bottom]>.tippy-arrow:after{border-bottom-color:#dee2e6;border-width:0 7px 7px;bottom:17px;left:1px}.tippy-box[data-theme~=quarto][data-placement^=bottom]>.tippy-svg-arrow>svg{bottom:15px}.tippy-box[data-theme~=quarto][data-placement^=bottom]>.tippy-svg-arrow:after{bottom:17px}.tippy-box[data-theme~=quarto][data-placement^=left]>.tippy-arrow:before{border-left-color:#fff}.tippy-box[data-theme~=quarto][data-placement^=left]>.tippy-arrow:after{border-left-color:#dee2e6;border-width:7px 0 7px 7px;left:17px;top:1px}.tippy-box[data-theme~=quarto][data-placement^=left]>.tippy-svg-arrow>svg{left:11px}.tippy-box[data-theme~=quarto][data-placement^=left]>.tippy-svg-arrow:after{left:12px}.tippy-box[data-theme~=quarto][data-placement^=right]>.tippy-arrow:before{border-right-color:#fff;right:16px}.tippy-box[data-theme~=quarto][data-placement^=right]>.tippy-arrow:after{border-width:7px 7px 7px 0;right:17px;top:1px;border-right-color:#dee2e6}.tippy-box[data-theme~=quarto][data-placement^=right]>.tippy-svg-arrow>svg{right:11px}.tippy-box[data-theme~=quarto][data-placement^=right]>.tippy-svg-arrow:after{right:12px}.tippy-box[data-theme~=quarto]>.tippy-svg-arrow{fill:#343a40}.tippy-box[data-theme~=quarto]>.tippy-svg-arrow:after{background-image:url(data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTYiIGhlaWdodD0iNiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48cGF0aCBkPSJNMCA2czEuNzk2LS4wMTMgNC42Ny0zLjYxNUM1Ljg1MS45IDYuOTMuMDA2IDggMGMxLjA3LS4wMDYgMi4xNDguODg3IDMuMzQzIDIuMzg1QzE0LjIzMyA2LjAwNSAxNiA2IDE2IDZIMHoiIGZpbGw9InJnYmEoMCwgOCwgMTYsIDAuMikiLz48L3N2Zz4=);background-size:16px 6px;width:16px;height:6px}.top-right{position:absolute;top:1em;right:1em}.visually-hidden{border:0;clip:rect(0 0 0 0);height:auto;margin:0;overflow:hidden;padding:0;position:absolute;width:1px;white-space:nowrap}.hidden{display:none !important}.zindex-bottom{z-index:-1 !important}figure.figure{display:block}.quarto-layout-panel{margin-bottom:1em}.quarto-layout-panel>figure{width:100%}.quarto-layout-panel>figure>figcaption,.quarto-layout-panel>.panel-caption{margin-top:10pt}.quarto-layout-panel>.table-caption{margin-top:0px}.table-caption p{margin-bottom:.5em}.quarto-layout-row{display:flex;flex-direction:row;align-items:flex-start}.quarto-layout-valign-top{align-items:flex-start}.quarto-layout-valign-bottom{align-items:flex-end}.quarto-layout-valign-center{align-items:center}.quarto-layout-cell{position:relative;margin-right:20px}.quarto-layout-cell:last-child{margin-right:0}.quarto-layout-cell figure,.quarto-layout-cell>p{margin:.2em}.quarto-layout-cell img{max-width:100%}.quarto-layout-cell .html-widget{width:100% !important}.quarto-layout-cell div figure p{margin:0}.quarto-layout-cell figure{display:block;margin-inline-start:0;margin-inline-end:0}.quarto-layout-cell table{display:inline-table}.quarto-layout-cell-subref figcaption,figure .quarto-layout-row figure figcaption{text-align:center;font-style:italic}.quarto-figure{position:relative;margin-bottom:1em}.quarto-figure>figure{width:100%;margin-bottom:0}.quarto-figure-left>figure>p,.quarto-figure-left>figure>div{text-align:left}.quarto-figure-center>figure>p,.quarto-figure-center>figure>div{text-align:center}.quarto-figure-right>figure>p,.quarto-figure-right>figure>div{text-align:right}.quarto-figure>figure>div.cell-annotation,.quarto-figure>figure>div code{text-align:left}figure>p:empty{display:none}figure>p:first-child{margin-top:0;margin-bottom:0}figure>figcaption.quarto-float-caption-bottom{margin-bottom:.5em}figure>figcaption.quarto-float-caption-top{margin-top:.5em}div[id^=tbl-]{position:relative}.quarto-figure>.anchorjs-link{position:absolute;top:.6em;right:.5em}div[id^=tbl-]>.anchorjs-link{position:absolute;top:.7em;right:.3em}.quarto-figure:hover>.anchorjs-link,div[id^=tbl-]:hover>.anchorjs-link,h2:hover>.anchorjs-link,.h2:hover>.anchorjs-link,h3:hover>.anchorjs-link,.h3:hover>.anchorjs-link,h4:hover>.anchorjs-link,.h4:hover>.anchorjs-link,h5:hover>.anchorjs-link,.h5:hover>.anchorjs-link,h6:hover>.anchorjs-link,.h6:hover>.anchorjs-link,.reveal-anchorjs-link>.anchorjs-link{opacity:1}#title-block-header{margin-block-end:1rem;position:relative;margin-top:-1px}#title-block-header .abstract{margin-block-start:1rem}#title-block-header .abstract .abstract-title{font-weight:600}#title-block-header a{text-decoration:none}#title-block-header .author,#title-block-header .date,#title-block-header .doi{margin-block-end:.2rem}#title-block-header .quarto-title-block>div{display:flex}#title-block-header .quarto-title-block>div>h1,#title-block-header .quarto-title-block>div>.h1{flex-grow:1}#title-block-header .quarto-title-block>div>button{flex-shrink:0;height:2.25rem;margin-top:0}@media(min-width: 992px){#title-block-header .quarto-title-block>div>button{margin-top:5px}}tr.header>th>p:last-of-type{margin-bottom:0px}table,table.table{margin-top:.5rem;margin-bottom:.5rem}caption,.table-caption{padding-top:.5rem;padding-bottom:.5rem;text-align:center}figure.quarto-float-tbl figcaption.quarto-float-caption-top{margin-top:.5rem;margin-bottom:.25rem;text-align:center}figure.quarto-float-tbl figcaption.quarto-float-caption-bottom{padding-top:.25rem;margin-bottom:.5rem;text-align:center}.utterances{max-width:none;margin-left:-8px}iframe{margin-bottom:1em}details{margin-bottom:1em}details[show]{margin-bottom:0}details>summary{color:#6c757d}details>summary>p:only-child{display:inline}pre.sourceCode,code.sourceCode{position:relative}dd code:not(.sourceCode),p code:not(.sourceCode){white-space:pre-wrap}code{white-space:pre}@media print{code{white-space:pre-wrap}}pre>code{display:block}pre>code.sourceCode{white-space:pre}pre>code.sourceCode>span>a:first-child::before{text-decoration:none}pre.code-overflow-wrap>code.sourceCode{white-space:pre-wrap}pre.code-overflow-scroll>code.sourceCode{white-space:pre}code a:any-link{color:inherit;text-decoration:none}code a:hover{color:inherit;text-decoration:underline}ul.task-list{padding-left:1em}[data-tippy-root]{display:inline-block}.tippy-content .footnote-back{display:none}.footnote-back{margin-left:.2em}.tippy-content{overflow-x:auto}.quarto-embedded-source-code{display:none}.quarto-unresolved-ref{font-weight:600}.quarto-cover-image{max-width:35%;float:right;margin-left:30px}.cell-output-display .widget-subarea{margin-bottom:1em}.cell-output-display:not(.no-overflow-x),.knitsql-table:not(.no-overflow-x){overflow-x:auto}.panel-input{margin-bottom:1em}.panel-input>div,.panel-input>div>div{display:inline-block;vertical-align:top;padding-right:12px}.panel-input>p:last-child{margin-bottom:0}.layout-sidebar{margin-bottom:1em}.layout-sidebar .tab-content{border:none}.tab-content>.page-columns.active{display:grid}div.sourceCode>iframe{width:100%;height:300px;margin-bottom:-0.5em}a{text-underline-offset:3px}.callout pre.sourceCode{padding-left:0}div.ansi-escaped-output{font-family:monospace;display:block}/*! -* -* ansi colors from IPython notebook's -* -* we also add `bright-[color]-` synonyms for the `-[color]-intense` classes since -* that seems to be what ansi_up emits -* -*/.ansi-black-fg{color:#3e424d}.ansi-black-bg{background-color:#3e424d}.ansi-black-intense-black,.ansi-bright-black-fg{color:#282c36}.ansi-black-intense-black,.ansi-bright-black-bg{background-color:#282c36}.ansi-red-fg{color:#e75c58}.ansi-red-bg{background-color:#e75c58}.ansi-red-intense-red,.ansi-bright-red-fg{color:#b22b31}.ansi-red-intense-red,.ansi-bright-red-bg{background-color:#b22b31}.ansi-green-fg{color:#00a250}.ansi-green-bg{background-color:#00a250}.ansi-green-intense-green,.ansi-bright-green-fg{color:#007427}.ansi-green-intense-green,.ansi-bright-green-bg{background-color:#007427}.ansi-yellow-fg{color:#ddb62b}.ansi-yellow-bg{background-color:#ddb62b}.ansi-yellow-intense-yellow,.ansi-bright-yellow-fg{color:#b27d12}.ansi-yellow-intense-yellow,.ansi-bright-yellow-bg{background-color:#b27d12}.ansi-blue-fg{color:#208ffb}.ansi-blue-bg{background-color:#208ffb}.ansi-blue-intense-blue,.ansi-bright-blue-fg{color:#0065ca}.ansi-blue-intense-blue,.ansi-bright-blue-bg{background-color:#0065ca}.ansi-magenta-fg{color:#d160c4}.ansi-magenta-bg{background-color:#d160c4}.ansi-magenta-intense-magenta,.ansi-bright-magenta-fg{color:#a03196}.ansi-magenta-intense-magenta,.ansi-bright-magenta-bg{background-color:#a03196}.ansi-cyan-fg{color:#60c6c8}.ansi-cyan-bg{background-color:#60c6c8}.ansi-cyan-intense-cyan,.ansi-bright-cyan-fg{color:#258f8f}.ansi-cyan-intense-cyan,.ansi-bright-cyan-bg{background-color:#258f8f}.ansi-white-fg{color:#c5c1b4}.ansi-white-bg{background-color:#c5c1b4}.ansi-white-intense-white,.ansi-bright-white-fg{color:#a1a6b2}.ansi-white-intense-white,.ansi-bright-white-bg{background-color:#a1a6b2}.ansi-default-inverse-fg{color:#fff}.ansi-default-inverse-bg{background-color:#000}.ansi-bold{font-weight:bold}.ansi-underline{text-decoration:underline}:root{--quarto-body-bg: #fff;--quarto-body-color: #343a40;--quarto-text-muted: #6c757d;--quarto-border-color: #dee2e6;--quarto-border-width: 1px}table.gt_table{color:var(--quarto-body-color);font-size:1em;width:100%;background-color:rgba(0,0,0,0);border-top-width:inherit;border-bottom-width:inherit;border-color:var(--quarto-border-color)}table.gt_table th.gt_column_spanner_outer{color:var(--quarto-body-color);background-color:rgba(0,0,0,0);border-top-width:inherit;border-bottom-width:inherit;border-color:var(--quarto-border-color)}table.gt_table th.gt_col_heading{color:var(--quarto-body-color);font-weight:bold;background-color:rgba(0,0,0,0)}table.gt_table thead.gt_col_headings{border-bottom:1px solid currentColor;border-top-width:inherit;border-top-color:var(--quarto-border-color)}table.gt_table thead.gt_col_headings:not(:first-child){border-top-width:1px;border-top-color:var(--quarto-border-color)}table.gt_table td.gt_row{border-bottom-width:1px;border-bottom-color:var(--quarto-border-color);border-top-width:0px}table.gt_table tbody.gt_table_body{border-top-width:1px;border-bottom-width:1px;border-bottom-color:var(--quarto-border-color);border-top-color:currentColor}div.columns{display:initial;gap:initial}div.column{display:inline-block;overflow-x:initial;vertical-align:top;width:50%}.code-annotation-tip-content{word-wrap:break-word}.code-annotation-container-hidden{display:none !important}dl.code-annotation-container-grid{display:grid;grid-template-columns:min-content auto}dl.code-annotation-container-grid dt{grid-column:1}dl.code-annotation-container-grid dd{grid-column:2}pre.sourceCode.code-annotation-code{padding-right:0}code.sourceCode .code-annotation-anchor{z-index:100;position:relative;float:right;background-color:rgba(0,0,0,0)}input[type=checkbox]{margin-right:.5ch}:root{--mermaid-bg-color: #fff;--mermaid-edge-color: #343a40;--mermaid-node-fg-color: #343a40;--mermaid-fg-color: #343a40;--mermaid-fg-color--lighter: #4b545c;--mermaid-fg-color--lightest: #626d78;--mermaid-font-family: Source Sans Pro, -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Helvetica Neue, Arial, sans-serif, Apple Color Emoji, Segoe UI Emoji, Segoe UI Symbol;--mermaid-label-bg-color: #fff;--mermaid-label-fg-color: #2780e3;--mermaid-node-bg-color: rgba(39, 128, 227, 0.1);--mermaid-node-fg-color: #343a40}@media print{:root{font-size:11pt}#quarto-sidebar,#TOC,.nav-page{display:none}.page-columns .content{grid-column-start:page-start}.fixed-top{position:relative}.panel-caption,.figure-caption,figcaption{color:#666}}.code-copy-button{position:absolute;top:0;right:0;border:0;margin-top:5px;margin-right:5px;background-color:rgba(0,0,0,0);z-index:3}.code-copy-button:focus{outline:none}.code-copy-button-tooltip{font-size:.75em}pre.sourceCode:hover>.code-copy-button>.bi::before{display:inline-block;height:1rem;width:1rem;content:"";vertical-align:-0.125em;background-image:url('data:image/svg+xml,');background-repeat:no-repeat;background-size:1rem 1rem}pre.sourceCode:hover>.code-copy-button-checked>.bi::before{background-image:url('data:image/svg+xml,')}pre.sourceCode:hover>.code-copy-button:hover>.bi::before{background-image:url('data:image/svg+xml,')}pre.sourceCode:hover>.code-copy-button-checked:hover>.bi::before{background-image:url('data:image/svg+xml,')}main ol ol,main ul ul,main ol ul,main ul ol{margin-bottom:1em}ul>li:not(:has(>p))>ul,ol>li:not(:has(>p))>ul,ul>li:not(:has(>p))>ol,ol>li:not(:has(>p))>ol{margin-bottom:0}ul>li:not(:has(>p))>ul>li:has(>p),ol>li:not(:has(>p))>ul>li:has(>p),ul>li:not(:has(>p))>ol>li:has(>p),ol>li:not(:has(>p))>ol>li:has(>p){margin-top:1rem}body{margin:0}main.page-columns>header>h1.title,main.page-columns>header>.title.h1{margin-bottom:0}@media(min-width: 992px){body .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start page-start-inset] 35px [body-start-outset] 35px [body-start] 1.5em [body-content-start] minmax(500px, calc(850px - 3em)) [body-content-end] 1.5em [body-end] 35px [body-end-outset] minmax(75px, 145px) [page-end-inset] 35px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.fullcontent:not(.floating):not(.docked) .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start page-start-inset] 35px [body-start-outset] 35px [body-start] 1.5em [body-content-start] minmax(500px, calc(850px - 3em)) [body-content-end] 1.5em [body-end] 35px [body-end-outset] 35px [page-end-inset page-end] 5fr [screen-end-inset] 1.5em}body.slimcontent:not(.floating):not(.docked) .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start page-start-inset] 35px [body-start-outset] 35px [body-start] 1.5em [body-content-start] minmax(500px, calc(850px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(0px, 200px) [page-end-inset] 35px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.listing:not(.floating):not(.docked) .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start] minmax(50px, 100px) [page-start-inset] 50px [body-start-outset] 50px [body-start] 1.5em [body-content-start] minmax(500px, calc(850px - 3em)) [body-content-end] 3em [body-end] 50px [body-end-outset] minmax(0px, 250px) [page-end-inset] minmax(50px, 100px) [page-end] 1fr [screen-end-inset] 1.5em [screen-end]}body:not(.floating):not(.docked) .page-columns.toc-left{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start] 35px [page-start-inset] minmax(0px, 175px) [body-start-outset] 35px [body-start] 1.5em [body-content-start] minmax(450px, calc(800px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(0px, 200px) [page-end-inset] 50px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body:not(.floating):not(.docked) .page-columns.toc-left .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start] 35px [page-start-inset] minmax(0px, 175px) [body-start-outset] 35px [body-start] 1.5em [body-content-start] minmax(450px, calc(800px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(0px, 200px) [page-end-inset] 50px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.floating .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start] minmax(25px, 50px) [page-start-inset] minmax(50px, 150px) [body-start-outset] minmax(25px, 50px) [body-start] 1.5em [body-content-start] minmax(500px, calc(800px - 3em)) [body-content-end] 1.5em [body-end] minmax(25px, 50px) [body-end-outset] minmax(50px, 150px) [page-end-inset] minmax(25px, 50px) [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.docked .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start] minmax(50px, 100px) [page-start-inset] 50px [body-start-outset] 50px [body-start] 1.5em [body-content-start] minmax(500px, calc(1000px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(50px, 100px) [page-end-inset] 50px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.docked.fullcontent .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start] minmax(50px, 100px) [page-start-inset] 50px [body-start-outset] 50px [body-start] 1.5em [body-content-start] minmax(500px, calc(1000px - 3em)) [body-content-end] 1.5em [body-end body-end-outset page-end-inset page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.floating.fullcontent .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start] 50px [page-start-inset] minmax(50px, 150px) [body-start-outset] 50px [body-start] 1.5em [body-content-start] minmax(500px, calc(800px - 3em)) [body-content-end] 1.5em [body-end body-end-outset page-end-inset page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.docked.slimcontent .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start] minmax(50px, 100px) [page-start-inset] 50px [body-start-outset] 50px [body-start] 1.5em [body-content-start] minmax(450px, calc(750px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(0px, 200px) [page-end-inset] 50px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.docked.listing .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start] minmax(50px, 100px) [page-start-inset] 50px [body-start-outset] 50px [body-start] 1.5em [body-content-start] minmax(500px, calc(1000px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(0px, 200px) [page-end-inset] 50px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.floating.slimcontent .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start] 50px [page-start-inset] minmax(50px, 150px) [body-start-outset] 50px [body-start] 1.5em [body-content-start] minmax(450px, calc(750px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(50px, 150px) [page-end-inset] 50px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.floating.listing .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start] minmax(25px, 50px) [page-start-inset] minmax(50px, 150px) [body-start-outset] minmax(25px, 50px) [body-start] 1.5em [body-content-start] minmax(500px, calc(800px - 3em)) [body-content-end] 1.5em [body-end] minmax(25px, 50px) [body-end-outset] minmax(50px, 150px) [page-end-inset] minmax(25px, 50px) [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}}@media(max-width: 991.98px){body .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset] 5fr [body-start] 1.5em [body-content-start] minmax(500px, calc(800px - 3em)) [body-content-end] 1.5em [body-end] 35px [body-end-outset] minmax(75px, 145px) [page-end-inset] 35px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.fullcontent:not(.floating):not(.docked) .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset] 5fr [body-start] 1.5em [body-content-start] minmax(500px, calc(800px - 3em)) [body-content-end] 1.5em [body-end body-end-outset page-end-inset page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.slimcontent:not(.floating):not(.docked) .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset] 5fr [body-start] 1.5em [body-content-start] minmax(500px, calc(800px - 3em)) [body-content-end] 1.5em [body-end] 35px [body-end-outset] minmax(75px, 145px) [page-end-inset] 35px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.listing:not(.floating):not(.docked) .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset] 5fr [body-start] 1.5em [body-content-start] minmax(500px, calc(1250px - 3em)) [body-content-end body-end body-end-outset page-end-inset page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body:not(.floating):not(.docked) .page-columns.toc-left{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start] 35px [page-start-inset] minmax(0px, 145px) [body-start-outset] 35px [body-start] 1.5em [body-content-start] minmax(450px, calc(800px - 3em)) [body-content-end] 1.5em [body-end body-end-outset page-end-inset page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body:not(.floating):not(.docked) .page-columns.toc-left .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start] 35px [page-start-inset] minmax(0px, 145px) [body-start-outset] 35px [body-start] 1.5em [body-content-start] minmax(450px, calc(800px - 3em)) [body-content-end] 1.5em [body-end body-end-outset page-end-inset page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.floating .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start page-start-inset body-start-outset body-start] 1.5em [body-content-start] minmax(500px, calc(750px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(75px, 150px) [page-end-inset] 25px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.docked .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset body-start body-content-start] minmax(500px, calc(750px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(25px, 50px) [page-end-inset] 50px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.docked.fullcontent .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset body-start body-content-start] minmax(500px, calc(1000px - 3em)) [body-content-end] 1.5em [body-end body-end-outset page-end-inset page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.floating.fullcontent .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start page-start-inset body-start-outset body-start] 1em [body-content-start] minmax(500px, calc(800px - 3em)) [body-content-end] 1.5em [body-end body-end-outset page-end-inset page-end] 4fr [screen-end-inset] 1.5em [screen-end]}body.docked.slimcontent .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset body-start body-content-start] minmax(500px, calc(750px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(25px, 50px) [page-end-inset] 50px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.docked.listing .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset body-start body-content-start] minmax(500px, calc(750px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(25px, 50px) [page-end-inset] 50px [page-end] 5fr [screen-end-inset] 1.5em [screen-end]}body.floating.slimcontent .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start page-start-inset body-start-outset body-start] 1em [body-content-start] minmax(500px, calc(750px - 3em)) [body-content-end] 1.5em [body-end] 35px [body-end-outset] minmax(75px, 145px) [page-end-inset] 35px [page-end] 4fr [screen-end-inset] 1.5em [screen-end]}body.floating.listing .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset] 5fr [page-start page-start-inset body-start-outset body-start] 1em [body-content-start] minmax(500px, calc(750px - 3em)) [body-content-end] 1.5em [body-end] 50px [body-end-outset] minmax(75px, 150px) [page-end-inset] 25px [page-end] 4fr [screen-end-inset] 1.5em [screen-end]}}@media(max-width: 767.98px){body .page-columns,body.fullcontent:not(.floating):not(.docked) .page-columns,body.slimcontent:not(.floating):not(.docked) .page-columns,body.docked .page-columns,body.docked.slimcontent .page-columns,body.docked.fullcontent .page-columns,body.floating .page-columns,body.floating.slimcontent .page-columns,body.floating.fullcontent .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset body-start body-content-start] minmax(0px, 1fr) [body-content-end body-end body-end-outset page-end-inset page-end screen-end-inset] 1.5em [screen-end]}body:not(.floating):not(.docked) .page-columns.toc-left{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset body-start body-content-start] minmax(0px, 1fr) [body-content-end body-end body-end-outset page-end-inset page-end screen-end-inset] 1.5em [screen-end]}body:not(.floating):not(.docked) .page-columns.toc-left .page-columns{display:grid;gap:0;grid-template-columns:[screen-start] 1.5em [screen-start-inset page-start page-start-inset body-start-outset body-start body-content-start] minmax(0px, 1fr) [body-content-end body-end body-end-outset page-end-inset page-end screen-end-inset] 1.5em [screen-end]}nav[role=doc-toc]{display:none}}body,.page-row-navigation{grid-template-rows:[page-top] max-content [contents-top] max-content [contents-bottom] max-content [page-bottom]}.page-rows-contents{grid-template-rows:[content-top] minmax(max-content, 1fr) [content-bottom] minmax(60px, max-content) [page-bottom]}.page-full{grid-column:screen-start/screen-end !important}.page-columns>*{grid-column:body-content-start/body-content-end}.page-columns.column-page>*{grid-column:page-start/page-end}.page-columns.column-page-left .page-columns.page-full>*,.page-columns.column-page-left>*{grid-column:page-start/body-content-end}.page-columns.column-page-right .page-columns.page-full>*,.page-columns.column-page-right>*{grid-column:body-content-start/page-end}.page-rows{grid-auto-rows:auto}.header{grid-column:screen-start/screen-end;grid-row:page-top/contents-top}#quarto-content{padding:0;grid-column:screen-start/screen-end;grid-row:contents-top/contents-bottom}body.floating .sidebar.sidebar-navigation{grid-column:page-start/body-start;grid-row:content-top/page-bottom}body.docked .sidebar.sidebar-navigation{grid-column:screen-start/body-start;grid-row:content-top/page-bottom}.sidebar.toc-left{grid-column:page-start/body-start;grid-row:content-top/page-bottom}.sidebar.margin-sidebar{grid-column:body-end/page-end;grid-row:content-top/page-bottom}.page-columns .content{grid-column:body-content-start/body-content-end;grid-row:content-top/content-bottom;align-content:flex-start}.page-columns .page-navigation{grid-column:body-content-start/body-content-end;grid-row:content-bottom/page-bottom}.page-columns .footer{grid-column:screen-start/screen-end;grid-row:contents-bottom/page-bottom}.page-columns .column-body{grid-column:body-content-start/body-content-end}.page-columns .column-body-fullbleed{grid-column:body-start/body-end}.page-columns .column-body-outset{grid-column:body-start-outset/body-end-outset;z-index:998;opacity:.999}.page-columns .column-body-outset table{background:#fff}.page-columns .column-body-outset-left{grid-column:body-start-outset/body-content-end;z-index:998;opacity:.999}.page-columns .column-body-outset-left table{background:#fff}.page-columns .column-body-outset-right{grid-column:body-content-start/body-end-outset;z-index:998;opacity:.999}.page-columns .column-body-outset-right table{background:#fff}.page-columns .column-page{grid-column:page-start/page-end;z-index:998;opacity:.999}.page-columns .column-page table{background:#fff}.page-columns .column-page-inset{grid-column:page-start-inset/page-end-inset;z-index:998;opacity:.999}.page-columns .column-page-inset table{background:#fff}.page-columns .column-page-inset-left{grid-column:page-start-inset/body-content-end;z-index:998;opacity:.999}.page-columns .column-page-inset-left table{background:#fff}.page-columns .column-page-inset-right{grid-column:body-content-start/page-end-inset;z-index:998;opacity:.999}.page-columns .column-page-inset-right figcaption table{background:#fff}.page-columns .column-page-left{grid-column:page-start/body-content-end;z-index:998;opacity:.999}.page-columns .column-page-left table{background:#fff}.page-columns .column-page-right{grid-column:body-content-start/page-end;z-index:998;opacity:.999}.page-columns .column-page-right figcaption table{background:#fff}#quarto-content.page-columns #quarto-margin-sidebar,#quarto-content.page-columns #quarto-sidebar{z-index:1}@media(max-width: 991.98px){#quarto-content.page-columns #quarto-margin-sidebar.collapse,#quarto-content.page-columns #quarto-sidebar.collapse,#quarto-content.page-columns #quarto-margin-sidebar.collapsing,#quarto-content.page-columns #quarto-sidebar.collapsing{z-index:1055}}#quarto-content.page-columns main.column-page,#quarto-content.page-columns main.column-page-right,#quarto-content.page-columns main.column-page-left{z-index:0}.page-columns .column-screen-inset{grid-column:screen-start-inset/screen-end-inset;z-index:998;opacity:.999}.page-columns .column-screen-inset table{background:#fff}.page-columns .column-screen-inset-left{grid-column:screen-start-inset/body-content-end;z-index:998;opacity:.999}.page-columns .column-screen-inset-left table{background:#fff}.page-columns .column-screen-inset-right{grid-column:body-content-start/screen-end-inset;z-index:998;opacity:.999}.page-columns .column-screen-inset-right table{background:#fff}.page-columns .column-screen{grid-column:screen-start/screen-end;z-index:998;opacity:.999}.page-columns .column-screen table{background:#fff}.page-columns .column-screen-left{grid-column:screen-start/body-content-end;z-index:998;opacity:.999}.page-columns .column-screen-left table{background:#fff}.page-columns .column-screen-right{grid-column:body-content-start/screen-end;z-index:998;opacity:.999}.page-columns .column-screen-right table{background:#fff}.page-columns .column-screen-inset-shaded{grid-column:screen-start/screen-end;padding:1em;background:#f8f9fa;z-index:998;opacity:.999;margin-bottom:1em}.zindex-content{z-index:998;opacity:.999}.zindex-modal{z-index:1055;opacity:.999}.zindex-over-content{z-index:999;opacity:.999}img.img-fluid.column-screen,img.img-fluid.column-screen-inset-shaded,img.img-fluid.column-screen-inset,img.img-fluid.column-screen-inset-left,img.img-fluid.column-screen-inset-right,img.img-fluid.column-screen-left,img.img-fluid.column-screen-right{width:100%}@media(min-width: 992px){.margin-caption,div.aside,aside:not(.footnotes):not(.sidebar),.column-margin{grid-column:body-end/page-end !important;z-index:998}.column-sidebar{grid-column:page-start/body-start !important;z-index:998}.column-leftmargin{grid-column:screen-start-inset/body-start !important;z-index:998}.no-row-height{height:1em;overflow:visible}}@media(max-width: 991.98px){.margin-caption,div.aside,aside:not(.footnotes):not(.sidebar),.column-margin{grid-column:body-end/page-end !important;z-index:998}.no-row-height{height:1em;overflow:visible}.page-columns.page-full{overflow:visible}.page-columns.toc-left .margin-caption,.page-columns.toc-left div.aside,.page-columns.toc-left aside:not(.footnotes):not(.sidebar),.page-columns.toc-left .column-margin{grid-column:body-content-start/body-content-end !important;z-index:998;opacity:.999}.page-columns.toc-left .no-row-height{height:initial;overflow:initial}}@media(max-width: 767.98px){.margin-caption,div.aside,aside:not(.footnotes):not(.sidebar),.column-margin{grid-column:body-content-start/body-content-end !important;z-index:998;opacity:.999}.no-row-height{height:initial;overflow:initial}#quarto-margin-sidebar{display:none}#quarto-sidebar-toc-left{display:none}.hidden-sm{display:none}}.panel-grid{display:grid;grid-template-rows:repeat(1, 1fr);grid-template-columns:repeat(24, 1fr);gap:1em}.panel-grid .g-col-1{grid-column:auto/span 1}.panel-grid .g-col-2{grid-column:auto/span 2}.panel-grid .g-col-3{grid-column:auto/span 3}.panel-grid .g-col-4{grid-column:auto/span 4}.panel-grid .g-col-5{grid-column:auto/span 5}.panel-grid .g-col-6{grid-column:auto/span 6}.panel-grid .g-col-7{grid-column:auto/span 7}.panel-grid .g-col-8{grid-column:auto/span 8}.panel-grid .g-col-9{grid-column:auto/span 9}.panel-grid .g-col-10{grid-column:auto/span 10}.panel-grid .g-col-11{grid-column:auto/span 11}.panel-grid .g-col-12{grid-column:auto/span 12}.panel-grid .g-col-13{grid-column:auto/span 13}.panel-grid .g-col-14{grid-column:auto/span 14}.panel-grid .g-col-15{grid-column:auto/span 15}.panel-grid .g-col-16{grid-column:auto/span 16}.panel-grid .g-col-17{grid-column:auto/span 17}.panel-grid .g-col-18{grid-column:auto/span 18}.panel-grid .g-col-19{grid-column:auto/span 19}.panel-grid .g-col-20{grid-column:auto/span 20}.panel-grid .g-col-21{grid-column:auto/span 21}.panel-grid .g-col-22{grid-column:auto/span 22}.panel-grid .g-col-23{grid-column:auto/span 23}.panel-grid .g-col-24{grid-column:auto/span 24}.panel-grid .g-start-1{grid-column-start:1}.panel-grid .g-start-2{grid-column-start:2}.panel-grid .g-start-3{grid-column-start:3}.panel-grid .g-start-4{grid-column-start:4}.panel-grid .g-start-5{grid-column-start:5}.panel-grid .g-start-6{grid-column-start:6}.panel-grid .g-start-7{grid-column-start:7}.panel-grid .g-start-8{grid-column-start:8}.panel-grid .g-start-9{grid-column-start:9}.panel-grid .g-start-10{grid-column-start:10}.panel-grid .g-start-11{grid-column-start:11}.panel-grid .g-start-12{grid-column-start:12}.panel-grid .g-start-13{grid-column-start:13}.panel-grid .g-start-14{grid-column-start:14}.panel-grid .g-start-15{grid-column-start:15}.panel-grid .g-start-16{grid-column-start:16}.panel-grid .g-start-17{grid-column-start:17}.panel-grid .g-start-18{grid-column-start:18}.panel-grid .g-start-19{grid-column-start:19}.panel-grid .g-start-20{grid-column-start:20}.panel-grid .g-start-21{grid-column-start:21}.panel-grid .g-start-22{grid-column-start:22}.panel-grid .g-start-23{grid-column-start:23}@media(min-width: 576px){.panel-grid .g-col-sm-1{grid-column:auto/span 1}.panel-grid .g-col-sm-2{grid-column:auto/span 2}.panel-grid .g-col-sm-3{grid-column:auto/span 3}.panel-grid .g-col-sm-4{grid-column:auto/span 4}.panel-grid .g-col-sm-5{grid-column:auto/span 5}.panel-grid .g-col-sm-6{grid-column:auto/span 6}.panel-grid .g-col-sm-7{grid-column:auto/span 7}.panel-grid .g-col-sm-8{grid-column:auto/span 8}.panel-grid .g-col-sm-9{grid-column:auto/span 9}.panel-grid .g-col-sm-10{grid-column:auto/span 10}.panel-grid .g-col-sm-11{grid-column:auto/span 11}.panel-grid .g-col-sm-12{grid-column:auto/span 12}.panel-grid .g-col-sm-13{grid-column:auto/span 13}.panel-grid .g-col-sm-14{grid-column:auto/span 14}.panel-grid .g-col-sm-15{grid-column:auto/span 15}.panel-grid .g-col-sm-16{grid-column:auto/span 16}.panel-grid .g-col-sm-17{grid-column:auto/span 17}.panel-grid .g-col-sm-18{grid-column:auto/span 18}.panel-grid .g-col-sm-19{grid-column:auto/span 19}.panel-grid .g-col-sm-20{grid-column:auto/span 20}.panel-grid .g-col-sm-21{grid-column:auto/span 21}.panel-grid .g-col-sm-22{grid-column:auto/span 22}.panel-grid .g-col-sm-23{grid-column:auto/span 23}.panel-grid .g-col-sm-24{grid-column:auto/span 24}.panel-grid .g-start-sm-1{grid-column-start:1}.panel-grid .g-start-sm-2{grid-column-start:2}.panel-grid .g-start-sm-3{grid-column-start:3}.panel-grid .g-start-sm-4{grid-column-start:4}.panel-grid .g-start-sm-5{grid-column-start:5}.panel-grid .g-start-sm-6{grid-column-start:6}.panel-grid .g-start-sm-7{grid-column-start:7}.panel-grid .g-start-sm-8{grid-column-start:8}.panel-grid .g-start-sm-9{grid-column-start:9}.panel-grid .g-start-sm-10{grid-column-start:10}.panel-grid .g-start-sm-11{grid-column-start:11}.panel-grid .g-start-sm-12{grid-column-start:12}.panel-grid .g-start-sm-13{grid-column-start:13}.panel-grid .g-start-sm-14{grid-column-start:14}.panel-grid .g-start-sm-15{grid-column-start:15}.panel-grid .g-start-sm-16{grid-column-start:16}.panel-grid .g-start-sm-17{grid-column-start:17}.panel-grid .g-start-sm-18{grid-column-start:18}.panel-grid .g-start-sm-19{grid-column-start:19}.panel-grid .g-start-sm-20{grid-column-start:20}.panel-grid .g-start-sm-21{grid-column-start:21}.panel-grid .g-start-sm-22{grid-column-start:22}.panel-grid .g-start-sm-23{grid-column-start:23}}@media(min-width: 768px){.panel-grid .g-col-md-1{grid-column:auto/span 1}.panel-grid .g-col-md-2{grid-column:auto/span 2}.panel-grid .g-col-md-3{grid-column:auto/span 3}.panel-grid .g-col-md-4{grid-column:auto/span 4}.panel-grid .g-col-md-5{grid-column:auto/span 5}.panel-grid .g-col-md-6{grid-column:auto/span 6}.panel-grid .g-col-md-7{grid-column:auto/span 7}.panel-grid .g-col-md-8{grid-column:auto/span 8}.panel-grid .g-col-md-9{grid-column:auto/span 9}.panel-grid .g-col-md-10{grid-column:auto/span 10}.panel-grid .g-col-md-11{grid-column:auto/span 11}.panel-grid .g-col-md-12{grid-column:auto/span 12}.panel-grid .g-col-md-13{grid-column:auto/span 13}.panel-grid .g-col-md-14{grid-column:auto/span 14}.panel-grid .g-col-md-15{grid-column:auto/span 15}.panel-grid .g-col-md-16{grid-column:auto/span 16}.panel-grid .g-col-md-17{grid-column:auto/span 17}.panel-grid .g-col-md-18{grid-column:auto/span 18}.panel-grid .g-col-md-19{grid-column:auto/span 19}.panel-grid .g-col-md-20{grid-column:auto/span 20}.panel-grid .g-col-md-21{grid-column:auto/span 21}.panel-grid .g-col-md-22{grid-column:auto/span 22}.panel-grid .g-col-md-23{grid-column:auto/span 23}.panel-grid .g-col-md-24{grid-column:auto/span 24}.panel-grid .g-start-md-1{grid-column-start:1}.panel-grid .g-start-md-2{grid-column-start:2}.panel-grid .g-start-md-3{grid-column-start:3}.panel-grid .g-start-md-4{grid-column-start:4}.panel-grid .g-start-md-5{grid-column-start:5}.panel-grid .g-start-md-6{grid-column-start:6}.panel-grid .g-start-md-7{grid-column-start:7}.panel-grid .g-start-md-8{grid-column-start:8}.panel-grid .g-start-md-9{grid-column-start:9}.panel-grid .g-start-md-10{grid-column-start:10}.panel-grid .g-start-md-11{grid-column-start:11}.panel-grid .g-start-md-12{grid-column-start:12}.panel-grid .g-start-md-13{grid-column-start:13}.panel-grid .g-start-md-14{grid-column-start:14}.panel-grid .g-start-md-15{grid-column-start:15}.panel-grid .g-start-md-16{grid-column-start:16}.panel-grid .g-start-md-17{grid-column-start:17}.panel-grid .g-start-md-18{grid-column-start:18}.panel-grid .g-start-md-19{grid-column-start:19}.panel-grid .g-start-md-20{grid-column-start:20}.panel-grid .g-start-md-21{grid-column-start:21}.panel-grid .g-start-md-22{grid-column-start:22}.panel-grid .g-start-md-23{grid-column-start:23}}@media(min-width: 992px){.panel-grid .g-col-lg-1{grid-column:auto/span 1}.panel-grid .g-col-lg-2{grid-column:auto/span 2}.panel-grid .g-col-lg-3{grid-column:auto/span 3}.panel-grid .g-col-lg-4{grid-column:auto/span 4}.panel-grid .g-col-lg-5{grid-column:auto/span 5}.panel-grid .g-col-lg-6{grid-column:auto/span 6}.panel-grid .g-col-lg-7{grid-column:auto/span 7}.panel-grid .g-col-lg-8{grid-column:auto/span 8}.panel-grid .g-col-lg-9{grid-column:auto/span 9}.panel-grid .g-col-lg-10{grid-column:auto/span 10}.panel-grid .g-col-lg-11{grid-column:auto/span 11}.panel-grid .g-col-lg-12{grid-column:auto/span 12}.panel-grid .g-col-lg-13{grid-column:auto/span 13}.panel-grid .g-col-lg-14{grid-column:auto/span 14}.panel-grid .g-col-lg-15{grid-column:auto/span 15}.panel-grid .g-col-lg-16{grid-column:auto/span 16}.panel-grid .g-col-lg-17{grid-column:auto/span 17}.panel-grid .g-col-lg-18{grid-column:auto/span 18}.panel-grid .g-col-lg-19{grid-column:auto/span 19}.panel-grid .g-col-lg-20{grid-column:auto/span 20}.panel-grid .g-col-lg-21{grid-column:auto/span 21}.panel-grid .g-col-lg-22{grid-column:auto/span 22}.panel-grid .g-col-lg-23{grid-column:auto/span 23}.panel-grid .g-col-lg-24{grid-column:auto/span 24}.panel-grid .g-start-lg-1{grid-column-start:1}.panel-grid .g-start-lg-2{grid-column-start:2}.panel-grid .g-start-lg-3{grid-column-start:3}.panel-grid .g-start-lg-4{grid-column-start:4}.panel-grid .g-start-lg-5{grid-column-start:5}.panel-grid .g-start-lg-6{grid-column-start:6}.panel-grid .g-start-lg-7{grid-column-start:7}.panel-grid .g-start-lg-8{grid-column-start:8}.panel-grid .g-start-lg-9{grid-column-start:9}.panel-grid .g-start-lg-10{grid-column-start:10}.panel-grid .g-start-lg-11{grid-column-start:11}.panel-grid .g-start-lg-12{grid-column-start:12}.panel-grid .g-start-lg-13{grid-column-start:13}.panel-grid .g-start-lg-14{grid-column-start:14}.panel-grid .g-start-lg-15{grid-column-start:15}.panel-grid .g-start-lg-16{grid-column-start:16}.panel-grid .g-start-lg-17{grid-column-start:17}.panel-grid .g-start-lg-18{grid-column-start:18}.panel-grid .g-start-lg-19{grid-column-start:19}.panel-grid .g-start-lg-20{grid-column-start:20}.panel-grid .g-start-lg-21{grid-column-start:21}.panel-grid .g-start-lg-22{grid-column-start:22}.panel-grid .g-start-lg-23{grid-column-start:23}}@media(min-width: 1200px){.panel-grid .g-col-xl-1{grid-column:auto/span 1}.panel-grid .g-col-xl-2{grid-column:auto/span 2}.panel-grid .g-col-xl-3{grid-column:auto/span 3}.panel-grid .g-col-xl-4{grid-column:auto/span 4}.panel-grid .g-col-xl-5{grid-column:auto/span 5}.panel-grid .g-col-xl-6{grid-column:auto/span 6}.panel-grid .g-col-xl-7{grid-column:auto/span 7}.panel-grid .g-col-xl-8{grid-column:auto/span 8}.panel-grid .g-col-xl-9{grid-column:auto/span 9}.panel-grid .g-col-xl-10{grid-column:auto/span 10}.panel-grid .g-col-xl-11{grid-column:auto/span 11}.panel-grid .g-col-xl-12{grid-column:auto/span 12}.panel-grid .g-col-xl-13{grid-column:auto/span 13}.panel-grid .g-col-xl-14{grid-column:auto/span 14}.panel-grid .g-col-xl-15{grid-column:auto/span 15}.panel-grid .g-col-xl-16{grid-column:auto/span 16}.panel-grid .g-col-xl-17{grid-column:auto/span 17}.panel-grid .g-col-xl-18{grid-column:auto/span 18}.panel-grid .g-col-xl-19{grid-column:auto/span 19}.panel-grid .g-col-xl-20{grid-column:auto/span 20}.panel-grid .g-col-xl-21{grid-column:auto/span 21}.panel-grid .g-col-xl-22{grid-column:auto/span 22}.panel-grid .g-col-xl-23{grid-column:auto/span 23}.panel-grid .g-col-xl-24{grid-column:auto/span 24}.panel-grid .g-start-xl-1{grid-column-start:1}.panel-grid .g-start-xl-2{grid-column-start:2}.panel-grid .g-start-xl-3{grid-column-start:3}.panel-grid .g-start-xl-4{grid-column-start:4}.panel-grid .g-start-xl-5{grid-column-start:5}.panel-grid .g-start-xl-6{grid-column-start:6}.panel-grid .g-start-xl-7{grid-column-start:7}.panel-grid .g-start-xl-8{grid-column-start:8}.panel-grid .g-start-xl-9{grid-column-start:9}.panel-grid .g-start-xl-10{grid-column-start:10}.panel-grid .g-start-xl-11{grid-column-start:11}.panel-grid .g-start-xl-12{grid-column-start:12}.panel-grid .g-start-xl-13{grid-column-start:13}.panel-grid .g-start-xl-14{grid-column-start:14}.panel-grid .g-start-xl-15{grid-column-start:15}.panel-grid .g-start-xl-16{grid-column-start:16}.panel-grid .g-start-xl-17{grid-column-start:17}.panel-grid .g-start-xl-18{grid-column-start:18}.panel-grid .g-start-xl-19{grid-column-start:19}.panel-grid .g-start-xl-20{grid-column-start:20}.panel-grid .g-start-xl-21{grid-column-start:21}.panel-grid .g-start-xl-22{grid-column-start:22}.panel-grid .g-start-xl-23{grid-column-start:23}}@media(min-width: 1400px){.panel-grid .g-col-xxl-1{grid-column:auto/span 1}.panel-grid .g-col-xxl-2{grid-column:auto/span 2}.panel-grid .g-col-xxl-3{grid-column:auto/span 3}.panel-grid .g-col-xxl-4{grid-column:auto/span 4}.panel-grid .g-col-xxl-5{grid-column:auto/span 5}.panel-grid .g-col-xxl-6{grid-column:auto/span 6}.panel-grid .g-col-xxl-7{grid-column:auto/span 7}.panel-grid .g-col-xxl-8{grid-column:auto/span 8}.panel-grid .g-col-xxl-9{grid-column:auto/span 9}.panel-grid .g-col-xxl-10{grid-column:auto/span 10}.panel-grid .g-col-xxl-11{grid-column:auto/span 11}.panel-grid .g-col-xxl-12{grid-column:auto/span 12}.panel-grid .g-col-xxl-13{grid-column:auto/span 13}.panel-grid .g-col-xxl-14{grid-column:auto/span 14}.panel-grid .g-col-xxl-15{grid-column:auto/span 15}.panel-grid .g-col-xxl-16{grid-column:auto/span 16}.panel-grid .g-col-xxl-17{grid-column:auto/span 17}.panel-grid .g-col-xxl-18{grid-column:auto/span 18}.panel-grid .g-col-xxl-19{grid-column:auto/span 19}.panel-grid .g-col-xxl-20{grid-column:auto/span 20}.panel-grid .g-col-xxl-21{grid-column:auto/span 21}.panel-grid .g-col-xxl-22{grid-column:auto/span 22}.panel-grid .g-col-xxl-23{grid-column:auto/span 23}.panel-grid .g-col-xxl-24{grid-column:auto/span 24}.panel-grid .g-start-xxl-1{grid-column-start:1}.panel-grid .g-start-xxl-2{grid-column-start:2}.panel-grid .g-start-xxl-3{grid-column-start:3}.panel-grid .g-start-xxl-4{grid-column-start:4}.panel-grid .g-start-xxl-5{grid-column-start:5}.panel-grid .g-start-xxl-6{grid-column-start:6}.panel-grid .g-start-xxl-7{grid-column-start:7}.panel-grid .g-start-xxl-8{grid-column-start:8}.panel-grid .g-start-xxl-9{grid-column-start:9}.panel-grid .g-start-xxl-10{grid-column-start:10}.panel-grid .g-start-xxl-11{grid-column-start:11}.panel-grid .g-start-xxl-12{grid-column-start:12}.panel-grid .g-start-xxl-13{grid-column-start:13}.panel-grid .g-start-xxl-14{grid-column-start:14}.panel-grid .g-start-xxl-15{grid-column-start:15}.panel-grid .g-start-xxl-16{grid-column-start:16}.panel-grid .g-start-xxl-17{grid-column-start:17}.panel-grid .g-start-xxl-18{grid-column-start:18}.panel-grid .g-start-xxl-19{grid-column-start:19}.panel-grid .g-start-xxl-20{grid-column-start:20}.panel-grid .g-start-xxl-21{grid-column-start:21}.panel-grid .g-start-xxl-22{grid-column-start:22}.panel-grid .g-start-xxl-23{grid-column-start:23}}main{margin-top:1em;margin-bottom:1em}h1,.h1,h2,.h2{color:inherit;margin-top:2rem;margin-bottom:1rem;font-weight:600}h1.title,.title.h1{margin-top:0}main.content>section:first-of-type>h2:first-child,main.content>section:first-of-type>.h2:first-child{margin-top:0}h2,.h2{border-bottom:1px solid #dee2e6;padding-bottom:.5rem}h3,.h3{font-weight:600}h3,.h3,h4,.h4{opacity:.9;margin-top:1.5rem}h5,.h5,h6,.h6{opacity:.9}.header-section-number{color:#6d7a86}.nav-link.active .header-section-number{color:inherit}mark,.mark{padding:0em}.panel-caption,.figure-caption,.subfigure-caption,.table-caption,figcaption,caption{font-size:.9rem;color:#6d7a86}.quarto-layout-cell[data-ref-parent] caption{color:#6d7a86}.column-margin figcaption,.margin-caption,div.aside,aside,.column-margin{color:#6d7a86;font-size:.825rem}.panel-caption.margin-caption{text-align:inherit}.column-margin.column-container p{margin-bottom:0}.column-margin.column-container>*:not(.collapse):first-child{padding-bottom:.5em;display:block}.column-margin.column-container>*:not(.collapse):not(:first-child){padding-top:.5em;padding-bottom:.5em;display:block}.column-margin.column-container>*.collapse:not(.show){display:none}@media(min-width: 768px){.column-margin.column-container .callout-margin-content:first-child{margin-top:4.5em}.column-margin.column-container .callout-margin-content-simple:first-child{margin-top:3.5em}}.margin-caption>*{padding-top:.5em;padding-bottom:.5em}@media(max-width: 767.98px){.quarto-layout-row{flex-direction:column}}.nav-tabs .nav-item{margin-top:1px;cursor:pointer}.tab-content{margin-top:0px;border-left:#dee2e6 1px solid;border-right:#dee2e6 1px solid;border-bottom:#dee2e6 1px solid;margin-left:0;padding:1em;margin-bottom:1em}@media(max-width: 767.98px){.layout-sidebar{margin-left:0;margin-right:0}}.panel-sidebar,.panel-sidebar .form-control,.panel-input,.panel-input .form-control,.selectize-dropdown{font-size:.9rem}.panel-sidebar .form-control,.panel-input .form-control{padding-top:.1rem}.tab-pane div.sourceCode{margin-top:0px}.tab-pane>p{padding-top:0}.tab-pane>p:nth-child(1){padding-top:0}.tab-pane>p:last-child{margin-bottom:0}.tab-pane>pre:last-child{margin-bottom:0}.tab-content>.tab-pane:not(.active){display:none !important}div.sourceCode{background-color:rgba(233,236,239,.65);border:1px solid rgba(233,236,239,.65)}pre.sourceCode{background-color:rgba(0,0,0,0)}pre.sourceCode{border:none;font-size:.875em;overflow:visible !important;padding:.4em}div.sourceCode{overflow-y:hidden}.callout div.sourceCode{margin-left:initial}.blockquote{font-size:inherit;padding-left:1rem;padding-right:1.5rem;color:#6d7a86}.blockquote h1:first-child,.blockquote .h1:first-child,.blockquote h2:first-child,.blockquote .h2:first-child,.blockquote h3:first-child,.blockquote .h3:first-child,.blockquote h4:first-child,.blockquote .h4:first-child,.blockquote h5:first-child,.blockquote .h5:first-child{margin-top:0}pre{background-color:initial;padding:initial;border:initial}p pre code:not(.sourceCode),li pre code:not(.sourceCode),pre code:not(.sourceCode){background-color:initial}p code:not(.sourceCode),li code:not(.sourceCode),td code:not(.sourceCode){background-color:#f8f9fa;padding:.2em}nav p code:not(.sourceCode),nav li code:not(.sourceCode),nav td code:not(.sourceCode){background-color:rgba(0,0,0,0);padding:0}td code:not(.sourceCode){white-space:pre-wrap}#quarto-embedded-source-code-modal>.modal-dialog{max-width:1000px;padding-left:1.75rem;padding-right:1.75rem}#quarto-embedded-source-code-modal>.modal-dialog>.modal-content>.modal-body{padding:0}#quarto-embedded-source-code-modal>.modal-dialog>.modal-content>.modal-body div.sourceCode{margin:0;padding:.2rem .2rem;border-radius:0px;border:none}#quarto-embedded-source-code-modal>.modal-dialog>.modal-content>.modal-header{padding:.7rem}.code-tools-button{font-size:1rem;padding:.15rem .15rem;margin-left:5px;color:#6c757d;background-color:rgba(0,0,0,0);transition:initial;cursor:pointer}.code-tools-button>.bi::before{display:inline-block;height:1rem;width:1rem;content:"";vertical-align:-0.125em;background-image:url('data:image/svg+xml,');background-repeat:no-repeat;background-size:1rem 1rem}.code-tools-button:hover>.bi::before{background-image:url('data:image/svg+xml,')}#quarto-embedded-source-code-modal .code-copy-button>.bi::before{background-image:url('data:image/svg+xml,')}#quarto-embedded-source-code-modal .code-copy-button-checked>.bi::before{background-image:url('data:image/svg+xml,')}.sidebar{will-change:top;transition:top 200ms linear;position:sticky;overflow-y:auto;padding-top:1.2em;max-height:100vh}.sidebar.toc-left,.sidebar.margin-sidebar{top:0px;padding-top:1em}.sidebar.quarto-banner-title-block-sidebar>*{padding-top:1.65em}figure .quarto-notebook-link{margin-top:.5em}.quarto-notebook-link{font-size:.75em;color:#6c757d;margin-bottom:1em;text-decoration:none;display:block}.quarto-notebook-link:hover{text-decoration:underline;color:#2761e3}.quarto-notebook-link::before{display:inline-block;height:.75rem;width:.75rem;margin-bottom:0em;margin-right:.25em;content:"";vertical-align:-0.125em;background-image:url('data:image/svg+xml,');background-repeat:no-repeat;background-size:.75rem .75rem}.toc-actions i.bi,.quarto-code-links i.bi,.quarto-other-links i.bi,.quarto-alternate-notebooks i.bi,.quarto-alternate-formats i.bi{margin-right:.4em;font-size:.8rem}.quarto-other-links-text-target .quarto-code-links i.bi,.quarto-other-links-text-target .quarto-other-links i.bi{margin-right:.2em}.quarto-other-formats-text-target .quarto-alternate-formats i.bi{margin-right:.1em}.toc-actions i.bi.empty,.quarto-code-links i.bi.empty,.quarto-other-links i.bi.empty,.quarto-alternate-notebooks i.bi.empty,.quarto-alternate-formats i.bi.empty{padding-left:1em}.quarto-notebook h2,.quarto-notebook .h2{border-bottom:none}.quarto-notebook .cell-container{display:flex}.quarto-notebook .cell-container .cell{flex-grow:4}.quarto-notebook .cell-container .cell-decorator{padding-top:1.5em;padding-right:1em;text-align:right}.quarto-notebook .cell-container.code-fold .cell-decorator{padding-top:3em}.quarto-notebook .cell-code code{white-space:pre-wrap}.quarto-notebook .cell .cell-output-stderr pre code,.quarto-notebook .cell .cell-output-stdout pre code{white-space:pre-wrap;overflow-wrap:anywhere}.toc-actions,.quarto-alternate-formats,.quarto-other-links,.quarto-code-links,.quarto-alternate-notebooks{padding-left:0em}.sidebar .toc-actions a,.sidebar .quarto-alternate-formats a,.sidebar .quarto-other-links a,.sidebar .quarto-code-links a,.sidebar .quarto-alternate-notebooks a,.sidebar nav[role=doc-toc] a{text-decoration:none}.sidebar .toc-actions a:hover,.sidebar .quarto-other-links a:hover,.sidebar .quarto-code-links a:hover,.sidebar .quarto-alternate-formats a:hover,.sidebar .quarto-alternate-notebooks a:hover{color:#2761e3}.sidebar .toc-actions h2,.sidebar .toc-actions .h2,.sidebar .quarto-code-links h2,.sidebar .quarto-code-links .h2,.sidebar .quarto-other-links h2,.sidebar .quarto-other-links .h2,.sidebar .quarto-alternate-notebooks h2,.sidebar .quarto-alternate-notebooks .h2,.sidebar .quarto-alternate-formats h2,.sidebar .quarto-alternate-formats .h2,.sidebar nav[role=doc-toc]>h2,.sidebar nav[role=doc-toc]>.h2{font-weight:500;margin-bottom:.2rem;margin-top:.3rem;font-family:inherit;border-bottom:0;padding-bottom:0;padding-top:0px}.sidebar .toc-actions>h2,.sidebar .toc-actions>.h2,.sidebar .quarto-code-links>h2,.sidebar .quarto-code-links>.h2,.sidebar .quarto-other-links>h2,.sidebar .quarto-other-links>.h2,.sidebar .quarto-alternate-notebooks>h2,.sidebar .quarto-alternate-notebooks>.h2,.sidebar .quarto-alternate-formats>h2,.sidebar .quarto-alternate-formats>.h2{font-size:.8rem}.sidebar nav[role=doc-toc]>h2,.sidebar nav[role=doc-toc]>.h2{font-size:.875rem}.sidebar nav[role=doc-toc]>ul a{border-left:1px solid #e9ecef;padding-left:.6rem}.sidebar .toc-actions h2>ul a,.sidebar .toc-actions .h2>ul a,.sidebar .quarto-code-links h2>ul a,.sidebar .quarto-code-links .h2>ul a,.sidebar .quarto-other-links h2>ul a,.sidebar .quarto-other-links .h2>ul a,.sidebar .quarto-alternate-notebooks h2>ul a,.sidebar .quarto-alternate-notebooks .h2>ul a,.sidebar .quarto-alternate-formats h2>ul a,.sidebar .quarto-alternate-formats .h2>ul a{border-left:none;padding-left:.6rem}.sidebar .toc-actions ul a:empty,.sidebar .quarto-code-links ul a:empty,.sidebar .quarto-other-links ul a:empty,.sidebar .quarto-alternate-notebooks ul a:empty,.sidebar .quarto-alternate-formats ul a:empty,.sidebar nav[role=doc-toc]>ul a:empty{display:none}.sidebar .toc-actions ul,.sidebar .quarto-code-links ul,.sidebar .quarto-other-links ul,.sidebar .quarto-alternate-notebooks ul,.sidebar .quarto-alternate-formats ul{padding-left:0;list-style:none}.sidebar nav[role=doc-toc] ul{list-style:none;padding-left:0;list-style:none}.sidebar nav[role=doc-toc]>ul{margin-left:.45em}.quarto-margin-sidebar nav[role=doc-toc]{padding-left:.5em}.sidebar .toc-actions>ul,.sidebar .quarto-code-links>ul,.sidebar .quarto-other-links>ul,.sidebar .quarto-alternate-notebooks>ul,.sidebar .quarto-alternate-formats>ul{font-size:.8rem}.sidebar nav[role=doc-toc]>ul{font-size:.875rem}.sidebar .toc-actions ul li a,.sidebar .quarto-code-links ul li a,.sidebar .quarto-other-links ul li a,.sidebar .quarto-alternate-notebooks ul li a,.sidebar .quarto-alternate-formats ul li a,.sidebar nav[role=doc-toc]>ul li a{line-height:1.1rem;padding-bottom:.2rem;padding-top:.2rem;color:inherit}.sidebar nav[role=doc-toc] ul>li>ul>li>a{padding-left:1.2em}.sidebar nav[role=doc-toc] ul>li>ul>li>ul>li>a{padding-left:2.4em}.sidebar nav[role=doc-toc] ul>li>ul>li>ul>li>ul>li>a{padding-left:3.6em}.sidebar nav[role=doc-toc] ul>li>ul>li>ul>li>ul>li>ul>li>a{padding-left:4.8em}.sidebar nav[role=doc-toc] ul>li>ul>li>ul>li>ul>li>ul>li>ul>li>a{padding-left:6em}.sidebar nav[role=doc-toc] ul>li>a.active,.sidebar nav[role=doc-toc] ul>li>ul>li>a.active{border-left:1px solid #2761e3;color:#2761e3 !important}.sidebar nav[role=doc-toc] ul>li>a:hover,.sidebar nav[role=doc-toc] ul>li>ul>li>a:hover{color:#2761e3 !important}kbd,.kbd{color:#343a40;background-color:#f8f9fa;border:1px solid;border-radius:5px;border-color:#dee2e6}.quarto-appendix-contents div.hanging-indent{margin-left:0em}.quarto-appendix-contents div.hanging-indent div.csl-entry{margin-left:1em;text-indent:-1em}.citation a,.footnote-ref{text-decoration:none}.footnotes ol{padding-left:1em}.tippy-content>*{margin-bottom:.7em}.tippy-content>*:last-child{margin-bottom:0}.callout{margin-top:1.25rem;margin-bottom:1.25rem;border-radius:.25rem;overflow-wrap:break-word}.callout .callout-title-container{overflow-wrap:anywhere}.callout.callout-style-simple{padding:.4em .7em;border-left:5px solid;border-right:1px solid #dee2e6;border-top:1px solid #dee2e6;border-bottom:1px solid #dee2e6}.callout.callout-style-default{border-left:5px solid;border-right:1px solid #dee2e6;border-top:1px solid #dee2e6;border-bottom:1px solid #dee2e6}.callout .callout-body-container{flex-grow:1}.callout.callout-style-simple .callout-body{font-size:.9rem;font-weight:400}.callout.callout-style-default .callout-body{font-size:.9rem;font-weight:400}.callout:not(.no-icon).callout-titled.callout-style-simple .callout-body{padding-left:1.6em}.callout.callout-titled>.callout-header{padding-top:.2em;margin-bottom:-0.2em}.callout.callout-style-simple>div.callout-header{border-bottom:none;font-size:.9rem;font-weight:600;opacity:75%}.callout.callout-style-default>div.callout-header{border-bottom:none;font-weight:600;opacity:85%;font-size:.9rem;padding-left:.5em;padding-right:.5em}.callout.callout-style-default .callout-body{padding-left:.5em;padding-right:.5em}.callout.callout-style-default .callout-body>:first-child{padding-top:.5rem;margin-top:0}.callout>div.callout-header[data-bs-toggle=collapse]{cursor:pointer}.callout.callout-style-default .callout-header[aria-expanded=false],.callout.callout-style-default .callout-header[aria-expanded=true]{padding-top:0px;margin-bottom:0px;align-items:center}.callout.callout-titled .callout-body>:last-child:not(.sourceCode),.callout.callout-titled .callout-body>div>:last-child:not(.sourceCode){padding-bottom:.5rem;margin-bottom:0}.callout:not(.callout-titled) .callout-body>:first-child,.callout:not(.callout-titled) .callout-body>div>:first-child{margin-top:.25rem}.callout:not(.callout-titled) .callout-body>:last-child,.callout:not(.callout-titled) .callout-body>div>:last-child{margin-bottom:.2rem}.callout.callout-style-simple .callout-icon::before,.callout.callout-style-simple .callout-toggle::before{height:1rem;width:1rem;display:inline-block;content:"";background-repeat:no-repeat;background-size:1rem 1rem}.callout.callout-style-default .callout-icon::before,.callout.callout-style-default .callout-toggle::before{height:.9rem;width:.9rem;display:inline-block;content:"";background-repeat:no-repeat;background-size:.9rem .9rem}.callout.callout-style-default .callout-toggle::before{margin-top:5px}.callout .callout-btn-toggle .callout-toggle::before{transition:transform .2s linear}.callout .callout-header[aria-expanded=false] .callout-toggle::before{transform:rotate(-90deg)}.callout .callout-header[aria-expanded=true] .callout-toggle::before{transform:none}.callout.callout-style-simple:not(.no-icon) div.callout-icon-container{padding-top:.2em;padding-right:.55em}.callout.callout-style-default:not(.no-icon) div.callout-icon-container{padding-top:.1em;padding-right:.35em}.callout.callout-style-default:not(.no-icon) div.callout-title-container{margin-top:-1px}.callout.callout-style-default.callout-caution:not(.no-icon) div.callout-icon-container{padding-top:.3em;padding-right:.35em}.callout>.callout-body>.callout-icon-container>.no-icon,.callout>.callout-header>.callout-icon-container>.no-icon{display:none}div.callout.callout{border-left-color:#6c757d}div.callout.callout-style-default>.callout-header{background-color:#6c757d}div.callout-note.callout{border-left-color:#2780e3}div.callout-note.callout-style-default>.callout-header{background-color:#e9f2fc}div.callout-note:not(.callout-titled) .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-note.callout-titled .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-note .callout-toggle::before{background-image:url('data:image/svg+xml,')}div.callout-tip.callout{border-left-color:#3fb618}div.callout-tip.callout-style-default>.callout-header{background-color:#ecf8e8}div.callout-tip:not(.callout-titled) .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-tip.callout-titled .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-tip .callout-toggle::before{background-image:url('data:image/svg+xml,')}div.callout-warning.callout{border-left-color:#ff7518}div.callout-warning.callout-style-default>.callout-header{background-color:#fff1e8}div.callout-warning:not(.callout-titled) .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-warning.callout-titled .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-warning .callout-toggle::before{background-image:url('data:image/svg+xml,')}div.callout-caution.callout{border-left-color:#f0ad4e}div.callout-caution.callout-style-default>.callout-header{background-color:#fef7ed}div.callout-caution:not(.callout-titled) .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-caution.callout-titled .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-caution .callout-toggle::before{background-image:url('data:image/svg+xml,')}div.callout-important.callout{border-left-color:#ff0039}div.callout-important.callout-style-default>.callout-header{background-color:#ffe6eb}div.callout-important:not(.callout-titled) .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-important.callout-titled .callout-icon::before{background-image:url('data:image/svg+xml,');}div.callout-important .callout-toggle::before{background-image:url('data:image/svg+xml,')}.quarto-toggle-container{display:flex;align-items:center}.quarto-reader-toggle .bi::before,.quarto-color-scheme-toggle .bi::before{display:inline-block;height:1rem;width:1rem;content:"";background-repeat:no-repeat;background-size:1rem 1rem}.sidebar-navigation{padding-left:20px}.navbar{background-color:#ca225e;color:#fefdfd}.navbar .quarto-color-scheme-toggle:not(.alternate) .bi::before{background-image:url('data:image/svg+xml,')}.navbar .quarto-color-scheme-toggle.alternate .bi::before{background-image:url('data:image/svg+xml,')}.sidebar-navigation .quarto-color-scheme-toggle:not(.alternate) .bi::before{background-image:url('data:image/svg+xml,')}.sidebar-navigation .quarto-color-scheme-toggle.alternate .bi::before{background-image:url('data:image/svg+xml,')}.quarto-sidebar-toggle{border-color:#dee2e6;border-bottom-left-radius:.25rem;border-bottom-right-radius:.25rem;border-style:solid;border-width:1px;overflow:hidden;border-top-width:0px;padding-top:0px !important}.quarto-sidebar-toggle-title{cursor:pointer;padding-bottom:2px;margin-left:.25em;text-align:center;font-weight:400;font-size:.775em}#quarto-content .quarto-sidebar-toggle{background:#fafafa}#quarto-content .quarto-sidebar-toggle-title{color:#343a40}.quarto-sidebar-toggle-icon{color:#dee2e6;margin-right:.5em;float:right;transition:transform .2s ease}.quarto-sidebar-toggle-icon::before{padding-top:5px}.quarto-sidebar-toggle.expanded .quarto-sidebar-toggle-icon{transform:rotate(-180deg)}.quarto-sidebar-toggle.expanded .quarto-sidebar-toggle-title{border-bottom:solid #dee2e6 1px}.quarto-sidebar-toggle-contents{background-color:#fff;padding-right:10px;padding-left:10px;margin-top:0px !important;transition:max-height .5s ease}.quarto-sidebar-toggle.expanded .quarto-sidebar-toggle-contents{padding-top:1em;padding-bottom:10px}@media(max-width: 767.98px){.sidebar-menu-container{padding-bottom:5em}}.quarto-sidebar-toggle:not(.expanded) .quarto-sidebar-toggle-contents{padding-top:0px !important;padding-bottom:0px}nav[role=doc-toc]{z-index:1020}#quarto-sidebar>*,nav[role=doc-toc]>*{transition:opacity .1s ease,border .1s ease}#quarto-sidebar.slow>*,nav[role=doc-toc].slow>*{transition:opacity .4s ease,border .4s ease}.quarto-color-scheme-toggle:not(.alternate).top-right .bi::before{background-image:url('data:image/svg+xml,')}.quarto-color-scheme-toggle.alternate.top-right .bi::before{background-image:url('data:image/svg+xml,')}#quarto-appendix.default{border-top:1px solid #dee2e6}#quarto-appendix.default{background-color:#fff;padding-top:1.5em;margin-top:2em;z-index:998}#quarto-appendix.default .quarto-appendix-heading{margin-top:0;line-height:1.4em;font-weight:600;opacity:.9;border-bottom:none;margin-bottom:0}#quarto-appendix.default .footnotes ol,#quarto-appendix.default .footnotes ol li>p:last-of-type,#quarto-appendix.default .quarto-appendix-contents>p:last-of-type{margin-bottom:0}#quarto-appendix.default .footnotes ol{margin-left:.5em}#quarto-appendix.default .quarto-appendix-secondary-label{margin-bottom:.4em}#quarto-appendix.default .quarto-appendix-bibtex{font-size:.7em;padding:1em;border:solid 1px #dee2e6;margin-bottom:1em}#quarto-appendix.default .quarto-appendix-bibtex code.sourceCode{white-space:pre-wrap}#quarto-appendix.default .quarto-appendix-citeas{font-size:.9em;padding:1em;border:solid 1px #dee2e6;margin-bottom:1em}#quarto-appendix.default .quarto-appendix-heading{font-size:1em !important}#quarto-appendix.default *[role=doc-endnotes]>ol,#quarto-appendix.default .quarto-appendix-contents>*:not(h2):not(.h2){font-size:.9em}#quarto-appendix.default section{padding-bottom:1.5em}#quarto-appendix.default section *[role=doc-endnotes],#quarto-appendix.default section>*:not(a){opacity:.9;word-wrap:break-word}.btn.btn-quarto,div.cell-output-display .btn-quarto{--bs-btn-color: #cacccd;--bs-btn-bg: #343a40;--bs-btn-border-color: #343a40;--bs-btn-hover-color: #cacccd;--bs-btn-hover-bg: #52585d;--bs-btn-hover-border-color: #484e53;--bs-btn-focus-shadow-rgb: 75, 80, 85;--bs-btn-active-color: #fff;--bs-btn-active-bg: #5d6166;--bs-btn-active-border-color: #484e53;--bs-btn-active-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);--bs-btn-disabled-color: #fff;--bs-btn-disabled-bg: #343a40;--bs-btn-disabled-border-color: #343a40}nav.quarto-secondary-nav.color-navbar{background-color:#ca225e;color:#fefdfd}nav.quarto-secondary-nav.color-navbar h1,nav.quarto-secondary-nav.color-navbar .h1,nav.quarto-secondary-nav.color-navbar .quarto-btn-toggle{color:#fefdfd}@media(max-width: 991.98px){body.nav-sidebar .quarto-title-banner{margin-bottom:0;padding-bottom:1em}body.nav-sidebar #title-block-header{margin-block-end:0}}p.subtitle{margin-top:.25em;margin-bottom:.5em}code a:any-link{color:inherit;text-decoration-color:#6c757d}/*! light */div.observablehq table thead tr th{background-color:var(--bs-body-bg)}input,button,select,optgroup,textarea{background-color:var(--bs-body-bg)}.code-annotated .code-copy-button{margin-right:1.25em;margin-top:0;padding-bottom:0;padding-top:3px}.code-annotation-gutter-bg{background-color:#fff}.code-annotation-gutter{background-color:rgba(233,236,239,.65)}.code-annotation-gutter,.code-annotation-gutter-bg{height:100%;width:calc(20px + .5em);position:absolute;top:0;right:0}dl.code-annotation-container-grid dt{margin-right:1em;margin-top:.25rem}dl.code-annotation-container-grid dt{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;color:#4b545c;border:solid #4b545c 1px;border-radius:50%;height:22px;width:22px;line-height:22px;font-size:11px;text-align:center;vertical-align:middle;text-decoration:none}dl.code-annotation-container-grid dt[data-target-cell]{cursor:pointer}dl.code-annotation-container-grid dt[data-target-cell].code-annotation-active{color:#fff;border:solid #aaa 1px;background-color:#aaa}pre.code-annotation-code{padding-top:0;padding-bottom:0}pre.code-annotation-code code{z-index:3}#code-annotation-line-highlight-gutter{width:100%;border-top:solid rgba(170,170,170,.2666666667) 1px;border-bottom:solid rgba(170,170,170,.2666666667) 1px;z-index:2;background-color:rgba(170,170,170,.1333333333)}#code-annotation-line-highlight{margin-left:-4em;width:calc(100% + 4em);border-top:solid rgba(170,170,170,.2666666667) 1px;border-bottom:solid rgba(170,170,170,.2666666667) 1px;z-index:2;background-color:rgba(170,170,170,.1333333333)}code.sourceCode .code-annotation-anchor.code-annotation-active{background-color:var(--quarto-hl-normal-color, #aaaaaa);border:solid var(--quarto-hl-normal-color, #aaaaaa) 1px;color:#e9ecef;font-weight:bolder}code.sourceCode .code-annotation-anchor{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;color:var(--quarto-hl-co-color);border:solid var(--quarto-hl-co-color) 1px;border-radius:50%;height:18px;width:18px;font-size:9px;margin-top:2px}code.sourceCode button.code-annotation-anchor{padding:2px;user-select:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;-o-user-select:none}code.sourceCode a.code-annotation-anchor{line-height:18px;text-align:center;vertical-align:middle;cursor:default;text-decoration:none}@media print{.page-columns .column-screen-inset{grid-column:page-start-inset/page-end-inset;z-index:998;opacity:.999}.page-columns .column-screen-inset table{background:#fff}.page-columns .column-screen-inset-left{grid-column:page-start-inset/body-content-end;z-index:998;opacity:.999}.page-columns .column-screen-inset-left table{background:#fff}.page-columns .column-screen-inset-right{grid-column:body-content-start/page-end-inset;z-index:998;opacity:.999}.page-columns .column-screen-inset-right table{background:#fff}.page-columns .column-screen{grid-column:page-start/page-end;z-index:998;opacity:.999}.page-columns .column-screen table{background:#fff}.page-columns .column-screen-left{grid-column:page-start/body-content-end;z-index:998;opacity:.999}.page-columns .column-screen-left table{background:#fff}.page-columns .column-screen-right{grid-column:body-content-start/page-end;z-index:998;opacity:.999}.page-columns .column-screen-right table{background:#fff}.page-columns .column-screen-inset-shaded{grid-column:page-start-inset/page-end-inset;padding:1em;background:#f8f9fa;z-index:998;opacity:.999;margin-bottom:1em}}.quarto-video{margin-bottom:1em}.table{border-top:1px solid #d6d8d9;border-bottom:1px solid #d6d8d9}.table>thead{border-top-width:0;border-bottom:1px solid #9a9da0}.table a{word-break:break-word}.table>:not(caption)>*>*{background-color:unset;color:unset}#quarto-document-content .crosstalk-input .checkbox input[type=checkbox],#quarto-document-content .crosstalk-input .checkbox-inline input[type=checkbox]{position:unset;margin-top:unset;margin-left:unset}#quarto-document-content .row{margin-left:unset;margin-right:unset}.quarto-xref{white-space:nowrap}#quarto-draft-alert{margin-top:0px;margin-bottom:0px;padding:.3em;text-align:center;font-size:.9em}#quarto-draft-alert i{margin-right:.3em}#quarto-back-to-top{z-index:1000}pre{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:0.875em;font-weight:400}pre code{font-family:inherit;font-size:inherit;font-weight:inherit}code{font-family:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;font-size:0.875em;font-weight:400}a{background-color:rgba(0,0,0,0);font-weight:400;text-decoration:underline}a.external:after{content:"";background-image:url('data:image/svg+xml,');background-size:contain;background-repeat:no-repeat;background-position:center center;margin-left:.2em;padding-right:.75em}div.sourceCode code a.external:after{content:none}a.external:after:hover{cursor:pointer}.quarto-ext-icon{display:inline-block;font-size:.75em;padding-left:.3em}.code-with-filename .code-with-filename-file{margin-bottom:0;padding-bottom:2px;padding-top:2px;padding-left:.7em;border:var(--quarto-border-width) solid var(--quarto-border-color);border-radius:var(--quarto-border-radius);border-bottom:0;border-bottom-left-radius:0%;border-bottom-right-radius:0%}.code-with-filename div.sourceCode,.reveal .code-with-filename div.sourceCode{margin-top:0;border-top-left-radius:0%;border-top-right-radius:0%}.code-with-filename .code-with-filename-file pre{margin-bottom:0}.code-with-filename .code-with-filename-file{background-color:rgba(219,219,219,.8)}.quarto-dark .code-with-filename .code-with-filename-file{background-color:#555}.code-with-filename .code-with-filename-file strong{font-weight:400}.quarto-title-banner{margin-bottom:1em;color:#fefdfd;background:#ca225e}.quarto-title-banner a{color:#fefdfd}.quarto-title-banner h1,.quarto-title-banner .h1,.quarto-title-banner h2,.quarto-title-banner .h2{color:#fefdfd}.quarto-title-banner .code-tools-button{color:#dcb9b9}.quarto-title-banner .code-tools-button:hover{color:#fefdfd}.quarto-title-banner .code-tools-button>.bi::before{background-image:url('data:image/svg+xml,')}.quarto-title-banner .code-tools-button:hover>.bi::before{background-image:url('data:image/svg+xml,')}.quarto-title-banner .quarto-title .title{font-weight:600}.quarto-title-banner .quarto-categories{margin-top:.75em}@media(min-width: 992px){.quarto-title-banner{padding-top:2.5em;padding-bottom:2.5em}}@media(max-width: 991.98px){.quarto-title-banner{padding-top:1em;padding-bottom:1em}}@media(max-width: 767.98px){body.hypothesis-enabled #title-block-header>*{padding-right:20px}}main.quarto-banner-title-block>section:first-child>h2,main.quarto-banner-title-block>section:first-child>.h2,main.quarto-banner-title-block>section:first-child>h3,main.quarto-banner-title-block>section:first-child>.h3,main.quarto-banner-title-block>section:first-child>h4,main.quarto-banner-title-block>section:first-child>.h4{margin-top:0}.quarto-title .quarto-categories{display:flex;flex-wrap:wrap;row-gap:.5em;column-gap:.4em;padding-bottom:.5em;margin-top:.75em}.quarto-title .quarto-categories .quarto-category{padding:.25em .75em;font-size:.65em;text-transform:uppercase;border:solid 1px;border-radius:.25rem;opacity:.6}.quarto-title .quarto-categories .quarto-category a{color:inherit}.quarto-title-meta-container{display:grid;grid-template-columns:1fr auto}.quarto-title-meta-column-end{display:flex;flex-direction:column;padding-left:1em}.quarto-title-meta-column-end a .bi{margin-right:.3em}#title-block-header.quarto-title-block.default .quarto-title-meta{display:grid;grid-template-columns:repeat(2, 1fr);grid-column-gap:1em}#title-block-header.quarto-title-block.default .quarto-title .title{margin-bottom:0}#title-block-header.quarto-title-block.default .quarto-title-author-orcid img{margin-top:-0.2em;height:.8em;width:.8em}#title-block-header.quarto-title-block.default .quarto-title-author-email{opacity:.7}#title-block-header.quarto-title-block.default .quarto-description p:last-of-type{margin-bottom:0}#title-block-header.quarto-title-block.default .quarto-title-meta-contents p,#title-block-header.quarto-title-block.default .quarto-title-authors p,#title-block-header.quarto-title-block.default .quarto-title-affiliations p{margin-bottom:.1em}#title-block-header.quarto-title-block.default .quarto-title-meta-heading{text-transform:uppercase;margin-top:1em;font-size:.8em;opacity:.8;font-weight:400}#title-block-header.quarto-title-block.default .quarto-title-meta-contents{font-size:.9em}#title-block-header.quarto-title-block.default .quarto-title-meta-contents p.affiliation:last-of-type{margin-bottom:.1em}#title-block-header.quarto-title-block.default p.affiliation{margin-bottom:.1em}#title-block-header.quarto-title-block.default .keywords,#title-block-header.quarto-title-block.default .description,#title-block-header.quarto-title-block.default .abstract{margin-top:0}#title-block-header.quarto-title-block.default .keywords>p,#title-block-header.quarto-title-block.default .description>p,#title-block-header.quarto-title-block.default .abstract>p{font-size:.9em}#title-block-header.quarto-title-block.default .keywords>p:last-of-type,#title-block-header.quarto-title-block.default .description>p:last-of-type,#title-block-header.quarto-title-block.default .abstract>p:last-of-type{margin-bottom:0}#title-block-header.quarto-title-block.default .keywords .block-title,#title-block-header.quarto-title-block.default .description .block-title,#title-block-header.quarto-title-block.default .abstract .block-title{margin-top:1em;text-transform:uppercase;font-size:.8em;opacity:.8;font-weight:400}#title-block-header.quarto-title-block.default .quarto-title-meta-author{display:grid;grid-template-columns:minmax(max-content, 1fr) 1fr;grid-column-gap:1em}.quarto-title-tools-only{display:flex;justify-content:right}body{-webkit-font-smoothing:antialiased}.badge.bg-light{color:#343a40}.progress .progress-bar{font-size:8px;line-height:8px}:root{--quarto-scss-export-gray-300: #dee2e6;--quarto-scss-export-gray-500: #adb5bd;--quarto-scss-export-gray-600: #6c757d;--quarto-scss-export-gray-800: #343a40;--quarto-scss-export-card-cap-bg: rgba(52, 58, 64, 0.25);--quarto-scss-export-border-color: #dee2e6;--quarto-scss-export-text-muted: #6c757d;--quarto-scss-export-white: #fff;--quarto-scss-export-gray-100: #f8f9fa;--quarto-scss-export-gray-200: #e9ecef;--quarto-scss-export-gray-400: #ced4da;--quarto-scss-export-gray-700: #495057;--quarto-scss-export-gray-900: #212529;--quarto-scss-export-black: #000;--quarto-scss-export-blue: #2780e3;--quarto-scss-export-indigo: #6610f2;--quarto-scss-export-purple: #613d7c;--quarto-scss-export-pink: #e83e8c;--quarto-scss-export-red: #ff0039;--quarto-scss-export-orange: #f0ad4e;--quarto-scss-export-yellow: #ff7518;--quarto-scss-export-green: #3fb618;--quarto-scss-export-teal: #20c997;--quarto-scss-export-cyan: #9954bb;--quarto-scss-export-primary: #2780e3;--quarto-scss-export-secondary: #343a40;--quarto-scss-export-success: #3fb618;--quarto-scss-export-info: #9954bb;--quarto-scss-export-warning: #ff7518;--quarto-scss-export-danger: #ff0039;--quarto-scss-export-light: #f8f9fa;--quarto-scss-export-dark: #343a40;--quarto-scss-export-body-color: #343a40;--quarto-scss-export-title-banner-color: ;--quarto-scss-export-title-banner-bg: ;--quarto-scss-export-btn-code-copy-color: #5E5E5E;--quarto-scss-export-btn-code-copy-color-active: #4758AB;--quarto-scss-export-sidebar-bg: #fff;--quarto-scss-export-navbar-bg: #CA225E;--quarto-scss-export-link-color: #2761e3;--quarto-scss-export-link-color-bg: transparent;--quarto-scss-export-code-color: #7d12ba;--quarto-scss-export-code-bg: #f8f9fa;--quarto-scss-export-toc-color: #2761e3;--quarto-scss-export-toc-active-border: #2761e3;--quarto-scss-export-toc-inactive-border: #e9ecef;--quarto-scss-export-navbar-default: #2780e3;--quarto-scss-export-navbar-hl-override: #fdfdff;--quarto-scss-export-btn-bg: #343a40;--quarto-scss-export-btn-fg: #cacccd;--quarto-scss-export-body-contrast-bg: #fff;--quarto-scss-export-body-contrast-color: #343a40;--quarto-scss-export-navbar-fg: #fefdfd;--quarto-scss-export-navbar-hl: #fdfdff;--quarto-scss-export-navbar-brand: #fefdfd;--quarto-scss-export-navbar-brand-hl: #fdfdff;--quarto-scss-export-navbar-toggler-border-color: rgba(254, 253, 253, 0);--quarto-scss-export-navbar-hover-color: rgba(253, 253, 255, 0.8);--quarto-scss-export-navbar-disabled-color: rgba(254, 253, 253, 0.75);--quarto-scss-export-sidebar-fg: #595959;--quarto-scss-export-title-block-color: #343a40;--quarto-scss-export-title-block-contast-color: #fff;--quarto-scss-export-footer-bg: #fff;--quarto-scss-export-footer-fg: #757575;--quarto-scss-export-popover-bg: #fff;--quarto-scss-export-input-bg: #fff;--quarto-scss-export-input-border-color: #dee2e6;--quarto-scss-export-code-annotation-higlight-color: rgba(170, 170, 170, 0.2666666667);--quarto-scss-export-code-annotation-higlight-bg: rgba(170, 170, 170, 0.1333333333);--quarto-scss-export-table-group-separator-color: #9a9da0;--quarto-scss-export-table-group-separator-color-lighter: #d6d8d9;--quarto-scss-export-link-decoration: underline;--quarto-scss-export-table-border-color: #dee2e6;--quarto-scss-export-sidebar-glass-bg: rgba(102, 102, 102, 0.4);--quarto-scss-export-color-contrast-dark: #000;--quarto-scss-export-color-contrast-light: #fff;--quarto-scss-export-blue-100: #d4e6f9;--quarto-scss-export-blue-200: #a9ccf4;--quarto-scss-export-blue-300: #7db3ee;--quarto-scss-export-blue-400: #5299e9;--quarto-scss-export-blue-500: #2780e3;--quarto-scss-export-blue-600: #1f66b6;--quarto-scss-export-blue-700: #174d88;--quarto-scss-export-blue-800: #10335b;--quarto-scss-export-blue-900: #081a2d;--quarto-scss-export-indigo-100: #e0cffc;--quarto-scss-export-indigo-200: #c29ffa;--quarto-scss-export-indigo-300: #a370f7;--quarto-scss-export-indigo-400: #8540f5;--quarto-scss-export-indigo-500: #6610f2;--quarto-scss-export-indigo-600: #520dc2;--quarto-scss-export-indigo-700: #3d0a91;--quarto-scss-export-indigo-800: #290661;--quarto-scss-export-indigo-900: #140330;--quarto-scss-export-purple-100: #dfd8e5;--quarto-scss-export-purple-200: #c0b1cb;--quarto-scss-export-purple-300: #a08bb0;--quarto-scss-export-purple-400: #816496;--quarto-scss-export-purple-500: #613d7c;--quarto-scss-export-purple-600: #4e3163;--quarto-scss-export-purple-700: #3a254a;--quarto-scss-export-purple-800: #271832;--quarto-scss-export-purple-900: #130c19;--quarto-scss-export-pink-100: #fad8e8;--quarto-scss-export-pink-200: #f6b2d1;--quarto-scss-export-pink-300: #f18bba;--quarto-scss-export-pink-400: #ed65a3;--quarto-scss-export-pink-500: #e83e8c;--quarto-scss-export-pink-600: #ba3270;--quarto-scss-export-pink-700: #8b2554;--quarto-scss-export-pink-800: #5d1938;--quarto-scss-export-pink-900: #2e0c1c;--quarto-scss-export-red-100: #ffccd7;--quarto-scss-export-red-200: #ff99b0;--quarto-scss-export-red-300: #ff6688;--quarto-scss-export-red-400: #ff3361;--quarto-scss-export-red-500: #ff0039;--quarto-scss-export-red-600: #cc002e;--quarto-scss-export-red-700: #990022;--quarto-scss-export-red-800: #660017;--quarto-scss-export-red-900: #33000b;--quarto-scss-export-orange-100: #fcefdc;--quarto-scss-export-orange-200: #f9deb8;--quarto-scss-export-orange-300: #f6ce95;--quarto-scss-export-orange-400: #f3bd71;--quarto-scss-export-orange-500: #f0ad4e;--quarto-scss-export-orange-600: #c08a3e;--quarto-scss-export-orange-700: #90682f;--quarto-scss-export-orange-800: #60451f;--quarto-scss-export-orange-900: #302310;--quarto-scss-export-yellow-100: #ffe3d1;--quarto-scss-export-yellow-200: #ffc8a3;--quarto-scss-export-yellow-300: #ffac74;--quarto-scss-export-yellow-400: #ff9146;--quarto-scss-export-yellow-500: #ff7518;--quarto-scss-export-yellow-600: #cc5e13;--quarto-scss-export-yellow-700: #99460e;--quarto-scss-export-yellow-800: #662f0a;--quarto-scss-export-yellow-900: #331705;--quarto-scss-export-green-100: #d9f0d1;--quarto-scss-export-green-200: #b2e2a3;--quarto-scss-export-green-300: #8cd374;--quarto-scss-export-green-400: #65c546;--quarto-scss-export-green-500: #3fb618;--quarto-scss-export-green-600: #329213;--quarto-scss-export-green-700: #266d0e;--quarto-scss-export-green-800: #19490a;--quarto-scss-export-green-900: #0d2405;--quarto-scss-export-teal-100: #d2f4ea;--quarto-scss-export-teal-200: #a6e9d5;--quarto-scss-export-teal-300: #79dfc1;--quarto-scss-export-teal-400: #4dd4ac;--quarto-scss-export-teal-500: #20c997;--quarto-scss-export-teal-600: #1aa179;--quarto-scss-export-teal-700: #13795b;--quarto-scss-export-teal-800: #0d503c;--quarto-scss-export-teal-900: #06281e;--quarto-scss-export-cyan-100: #ebddf1;--quarto-scss-export-cyan-200: #d6bbe4;--quarto-scss-export-cyan-300: #c298d6;--quarto-scss-export-cyan-400: #ad76c9;--quarto-scss-export-cyan-500: #9954bb;--quarto-scss-export-cyan-600: #7a4396;--quarto-scss-export-cyan-700: #5c3270;--quarto-scss-export-cyan-800: #3d224b;--quarto-scss-export-cyan-900: #1f1125;--quarto-scss-export-default: #343a40;--quarto-scss-export-primary-text-emphasis: #10335b;--quarto-scss-export-secondary-text-emphasis: #15171a;--quarto-scss-export-success-text-emphasis: #19490a;--quarto-scss-export-info-text-emphasis: #3d224b;--quarto-scss-export-warning-text-emphasis: #662f0a;--quarto-scss-export-danger-text-emphasis: #660017;--quarto-scss-export-light-text-emphasis: #495057;--quarto-scss-export-dark-text-emphasis: #495057;--quarto-scss-export-primary-bg-subtle: #d4e6f9;--quarto-scss-export-secondary-bg-subtle: #d6d8d9;--quarto-scss-export-success-bg-subtle: #d9f0d1;--quarto-scss-export-info-bg-subtle: #ebddf1;--quarto-scss-export-warning-bg-subtle: #ffe3d1;--quarto-scss-export-danger-bg-subtle: #ffccd7;--quarto-scss-export-light-bg-subtle: #fcfcfd;--quarto-scss-export-dark-bg-subtle: #ced4da;--quarto-scss-export-primary-border-subtle: #a9ccf4;--quarto-scss-export-secondary-border-subtle: #aeb0b3;--quarto-scss-export-success-border-subtle: #b2e2a3;--quarto-scss-export-info-border-subtle: #d6bbe4;--quarto-scss-export-warning-border-subtle: #ffc8a3;--quarto-scss-export-danger-border-subtle: #ff99b0;--quarto-scss-export-light-border-subtle: #e9ecef;--quarto-scss-export-dark-border-subtle: #adb5bd;--quarto-scss-export-body-text-align: ;--quarto-scss-export-body-bg: #fff;--quarto-scss-export-body-secondary-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-body-secondary-bg: #e9ecef;--quarto-scss-export-body-tertiary-color: rgba(52, 58, 64, 0.5);--quarto-scss-export-body-tertiary-bg: #f8f9fa;--quarto-scss-export-body-emphasis-color: #000;--quarto-scss-export-link-hover-color: #1f4eb6;--quarto-scss-export-link-hover-decoration: ;--quarto-scss-export-border-color-translucent: rgba(0, 0, 0, 0.175);--quarto-scss-export-component-active-bg: #2780e3;--quarto-scss-export-component-active-color: #fff;--quarto-scss-export-focus-ring-color: rgba(39, 128, 227, 0.25);--quarto-scss-export-headings-font-family: ;--quarto-scss-export-headings-font-style: ;--quarto-scss-export-display-font-family: ;--quarto-scss-export-display-font-style: ;--quarto-scss-export-blockquote-footer-color: #6c757d;--quarto-scss-export-blockquote-border-color: #e9ecef;--quarto-scss-export-hr-bg-color: ;--quarto-scss-export-hr-height: ;--quarto-scss-export-hr-border-color: ;--quarto-scss-export-legend-font-weight: ;--quarto-scss-export-mark-bg: #ffe3d1;--quarto-scss-export-table-color: #343a40;--quarto-scss-export-table-bg: #fff;--quarto-scss-export-table-accent-bg: transparent;--quarto-scss-export-table-th-font-weight: ;--quarto-scss-export-table-striped-color: #343a40;--quarto-scss-export-table-striped-bg: rgba(0, 0, 0, 0.05);--quarto-scss-export-table-active-color: #343a40;--quarto-scss-export-table-active-bg: rgba(0, 0, 0, 0.1);--quarto-scss-export-table-hover-color: #343a40;--quarto-scss-export-table-hover-bg: rgba(0, 0, 0, 0.075);--quarto-scss-export-table-caption-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-input-btn-font-family: ;--quarto-scss-export-input-btn-focus-color: rgba(39, 128, 227, 0.25);--quarto-scss-export-btn-color: #343a40;--quarto-scss-export-btn-font-family: ;--quarto-scss-export-btn-white-space: ;--quarto-scss-export-btn-link-color: #2761e3;--quarto-scss-export-btn-link-hover-color: #1f4eb6;--quarto-scss-export-btn-link-disabled-color: #6c757d;--quarto-scss-export-form-text-font-style: ;--quarto-scss-export-form-text-font-weight: ;--quarto-scss-export-form-text-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-form-label-font-size: ;--quarto-scss-export-form-label-font-style: ;--quarto-scss-export-form-label-font-weight: ;--quarto-scss-export-form-label-color: ;--quarto-scss-export-input-font-family: ;--quarto-scss-export-input-disabled-color: ;--quarto-scss-export-input-disabled-bg: #e9ecef;--quarto-scss-export-input-disabled-border-color: ;--quarto-scss-export-input-color: #343a40;--quarto-scss-export-input-focus-bg: #fff;--quarto-scss-export-input-focus-border-color: #93c0f1;--quarto-scss-export-input-focus-color: #343a40;--quarto-scss-export-input-placeholder-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-input-plaintext-color: #343a40;--quarto-scss-export-form-check-label-color: ;--quarto-scss-export-form-check-transition: ;--quarto-scss-export-form-check-input-bg: #fff;--quarto-scss-export-form-check-input-focus-border: #93c0f1;--quarto-scss-export-form-check-input-checked-color: #fff;--quarto-scss-export-form-check-input-checked-bg-color: #2780e3;--quarto-scss-export-form-check-input-checked-border-color: #2780e3;--quarto-scss-export-form-check-input-indeterminate-color: #fff;--quarto-scss-export-form-check-input-indeterminate-bg-color: #2780e3;--quarto-scss-export-form-check-input-indeterminate-border-color: #2780e3;--quarto-scss-export-form-switch-color: rgba(0, 0, 0, 0.25);--quarto-scss-export-form-switch-focus-color: #93c0f1;--quarto-scss-export-form-switch-checked-color: #fff;--quarto-scss-export-input-group-addon-color: #343a40;--quarto-scss-export-input-group-addon-bg: #f8f9fa;--quarto-scss-export-input-group-addon-border-color: #dee2e6;--quarto-scss-export-form-select-font-family: ;--quarto-scss-export-form-select-color: #343a40;--quarto-scss-export-form-select-bg: #fff;--quarto-scss-export-form-select-disabled-color: ;--quarto-scss-export-form-select-disabled-bg: #e9ecef;--quarto-scss-export-form-select-disabled-border-color: ;--quarto-scss-export-form-select-indicator-color: #343a40;--quarto-scss-export-form-select-border-color: #dee2e6;--quarto-scss-export-form-select-focus-border-color: #93c0f1;--quarto-scss-export-form-range-track-bg: #f8f9fa;--quarto-scss-export-form-range-thumb-bg: #2780e3;--quarto-scss-export-form-range-thumb-active-bg: #bed9f7;--quarto-scss-export-form-range-thumb-disabled-bg: rgba(52, 58, 64, 0.75);--quarto-scss-export-form-file-button-color: #343a40;--quarto-scss-export-form-file-button-bg: #f8f9fa;--quarto-scss-export-form-file-button-hover-bg: #e9ecef;--quarto-scss-export-form-floating-label-disabled-color: #6c757d;--quarto-scss-export-form-feedback-font-style: ;--quarto-scss-export-form-feedback-valid-color: #3fb618;--quarto-scss-export-form-feedback-invalid-color: #ff0039;--quarto-scss-export-form-feedback-icon-valid-color: #3fb618;--quarto-scss-export-form-feedback-icon-invalid-color: #ff0039;--quarto-scss-export-form-valid-color: #3fb618;--quarto-scss-export-form-valid-border-color: #3fb618;--quarto-scss-export-form-invalid-color: #ff0039;--quarto-scss-export-form-invalid-border-color: #ff0039;--quarto-scss-export-nav-link-font-size: ;--quarto-scss-export-nav-link-font-weight: ;--quarto-scss-export-nav-link-color: #2761e3;--quarto-scss-export-nav-link-hover-color: #1f4eb6;--quarto-scss-export-nav-link-disabled-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-nav-tabs-border-color: #dee2e6;--quarto-scss-export-nav-tabs-link-hover-border-color: #e9ecef #e9ecef #dee2e6;--quarto-scss-export-nav-tabs-link-active-color: #000;--quarto-scss-export-nav-tabs-link-active-bg: #fff;--quarto-scss-export-nav-pills-link-active-bg: #2780e3;--quarto-scss-export-nav-pills-link-active-color: #fff;--quarto-scss-export-nav-underline-link-active-color: #000;--quarto-scss-export-navbar-padding-x: ;--quarto-scss-export-navbar-light-contrast: #fff;--quarto-scss-export-navbar-dark-contrast: #fff;--quarto-scss-export-navbar-light-icon-color: rgba(255, 255, 255, 0.75);--quarto-scss-export-navbar-dark-icon-color: rgba(255, 255, 255, 0.75);--quarto-scss-export-dropdown-color: #343a40;--quarto-scss-export-dropdown-bg: #fff;--quarto-scss-export-dropdown-border-color: rgba(0, 0, 0, 0.175);--quarto-scss-export-dropdown-divider-bg: rgba(0, 0, 0, 0.175);--quarto-scss-export-dropdown-link-color: #343a40;--quarto-scss-export-dropdown-link-hover-color: #343a40;--quarto-scss-export-dropdown-link-hover-bg: #f8f9fa;--quarto-scss-export-dropdown-link-active-bg: #2780e3;--quarto-scss-export-dropdown-link-active-color: #fff;--quarto-scss-export-dropdown-link-disabled-color: rgba(52, 58, 64, 0.5);--quarto-scss-export-dropdown-header-color: #6c757d;--quarto-scss-export-dropdown-dark-color: #dee2e6;--quarto-scss-export-dropdown-dark-bg: #343a40;--quarto-scss-export-dropdown-dark-border-color: rgba(0, 0, 0, 0.175);--quarto-scss-export-dropdown-dark-divider-bg: rgba(0, 0, 0, 0.175);--quarto-scss-export-dropdown-dark-box-shadow: ;--quarto-scss-export-dropdown-dark-link-color: #dee2e6;--quarto-scss-export-dropdown-dark-link-hover-color: #fff;--quarto-scss-export-dropdown-dark-link-hover-bg: rgba(255, 255, 255, 0.15);--quarto-scss-export-dropdown-dark-link-active-color: #fff;--quarto-scss-export-dropdown-dark-link-active-bg: #2780e3;--quarto-scss-export-dropdown-dark-link-disabled-color: #adb5bd;--quarto-scss-export-dropdown-dark-header-color: #adb5bd;--quarto-scss-export-pagination-color: #2761e3;--quarto-scss-export-pagination-bg: #fff;--quarto-scss-export-pagination-border-color: #dee2e6;--quarto-scss-export-pagination-focus-color: #1f4eb6;--quarto-scss-export-pagination-focus-bg: #e9ecef;--quarto-scss-export-pagination-hover-color: #1f4eb6;--quarto-scss-export-pagination-hover-bg: #f8f9fa;--quarto-scss-export-pagination-hover-border-color: #dee2e6;--quarto-scss-export-pagination-active-color: #fff;--quarto-scss-export-pagination-active-bg: #2780e3;--quarto-scss-export-pagination-active-border-color: #2780e3;--quarto-scss-export-pagination-disabled-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-pagination-disabled-bg: #e9ecef;--quarto-scss-export-pagination-disabled-border-color: #dee2e6;--quarto-scss-export-card-title-color: ;--quarto-scss-export-card-subtitle-color: ;--quarto-scss-export-card-border-color: rgba(0, 0, 0, 0.175);--quarto-scss-export-card-box-shadow: ;--quarto-scss-export-card-cap-color: ;--quarto-scss-export-card-height: ;--quarto-scss-export-card-color: ;--quarto-scss-export-card-bg: #fff;--quarto-scss-export-accordion-color: #343a40;--quarto-scss-export-accordion-bg: #fff;--quarto-scss-export-accordion-border-color: #dee2e6;--quarto-scss-export-accordion-button-color: #343a40;--quarto-scss-export-accordion-button-bg: #fff;--quarto-scss-export-accordion-button-active-bg: #d4e6f9;--quarto-scss-export-accordion-button-active-color: #10335b;--quarto-scss-export-accordion-button-focus-border-color: #93c0f1;--quarto-scss-export-accordion-icon-color: #343a40;--quarto-scss-export-accordion-icon-active-color: #10335b;--quarto-scss-export-tooltip-color: #fff;--quarto-scss-export-tooltip-bg: #000;--quarto-scss-export-tooltip-margin: ;--quarto-scss-export-tooltip-arrow-color: ;--quarto-scss-export-form-feedback-tooltip-line-height: ;--quarto-scss-export-popover-border-color: rgba(0, 0, 0, 0.175);--quarto-scss-export-popover-header-bg: #e9ecef;--quarto-scss-export-popover-body-color: #343a40;--quarto-scss-export-popover-arrow-color: #fff;--quarto-scss-export-popover-arrow-outer-color: rgba(0, 0, 0, 0.175);--quarto-scss-export-toast-color: ;--quarto-scss-export-toast-background-color: rgba(255, 255, 255, 0.85);--quarto-scss-export-toast-border-color: rgba(0, 0, 0, 0.175);--quarto-scss-export-toast-header-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-toast-header-background-color: rgba(255, 255, 255, 0.85);--quarto-scss-export-toast-header-border-color: rgba(0, 0, 0, 0.175);--quarto-scss-export-badge-color: #fff;--quarto-scss-export-modal-content-color: ;--quarto-scss-export-modal-content-bg: #fff;--quarto-scss-export-modal-content-border-color: rgba(0, 0, 0, 0.175);--quarto-scss-export-modal-backdrop-bg: #000;--quarto-scss-export-modal-header-border-color: #dee2e6;--quarto-scss-export-modal-footer-bg: ;--quarto-scss-export-modal-footer-border-color: #dee2e6;--quarto-scss-export-progress-bg: #e9ecef;--quarto-scss-export-progress-bar-color: #fff;--quarto-scss-export-progress-bar-bg: #2780e3;--quarto-scss-export-list-group-color: #343a40;--quarto-scss-export-list-group-bg: #fff;--quarto-scss-export-list-group-border-color: #dee2e6;--quarto-scss-export-list-group-hover-bg: #f8f9fa;--quarto-scss-export-list-group-active-bg: #2780e3;--quarto-scss-export-list-group-active-color: #fff;--quarto-scss-export-list-group-active-border-color: #2780e3;--quarto-scss-export-list-group-disabled-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-list-group-disabled-bg: #fff;--quarto-scss-export-list-group-action-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-list-group-action-hover-color: #000;--quarto-scss-export-list-group-action-active-color: #343a40;--quarto-scss-export-list-group-action-active-bg: #e9ecef;--quarto-scss-export-thumbnail-bg: #fff;--quarto-scss-export-thumbnail-border-color: #dee2e6;--quarto-scss-export-figure-caption-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-breadcrumb-font-size: ;--quarto-scss-export-breadcrumb-bg: ;--quarto-scss-export-breadcrumb-divider-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-breadcrumb-active-color: rgba(52, 58, 64, 0.75);--quarto-scss-export-breadcrumb-border-radius: ;--quarto-scss-export-carousel-control-color: #fff;--quarto-scss-export-carousel-indicator-active-bg: #fff;--quarto-scss-export-carousel-caption-color: #fff;--quarto-scss-export-carousel-dark-indicator-active-bg: #000;--quarto-scss-export-carousel-dark-caption-color: #000;--quarto-scss-export-btn-close-color: #000;--quarto-scss-export-offcanvas-border-color: rgba(0, 0, 0, 0.175);--quarto-scss-export-offcanvas-bg-color: #fff;--quarto-scss-export-offcanvas-color: #343a40;--quarto-scss-export-offcanvas-backdrop-bg: #000;--quarto-scss-export-code-color-dark: white;--quarto-scss-export-kbd-color: #fff;--quarto-scss-export-kbd-bg: #343a40;--quarto-scss-export-nested-kbd-font-weight: ;--quarto-scss-export-pre-bg: #f8f9fa;--quarto-scss-export-pre-color: #000;--quarto-scss-export-bslib-page-sidebar-title-bg: #CA225E;--quarto-scss-export-bslib-page-sidebar-title-color: #fff;--quarto-scss-export-bslib-sidebar-bg: rgba(var(--bs-emphasis-color-rgb, 0, 0, 0), 0.05);--quarto-scss-export-bslib-sidebar-toggle-bg: rgba(var(--bs-emphasis-color-rgb, 0, 0, 0), 0.1);--quarto-scss-export-sidebar-color: #595959;--quarto-scss-export-sidebar-hover-color: rgba(33, 81, 191, 0.8);--quarto-scss-export-sidebar-disabled-color: rgba(89, 89, 89, 0.75);--quarto-scss-export-valuebox-bg-primary: #5397e9;--quarto-scss-export-valuebox-bg-secondary: #343a40;--quarto-scss-export-valuebox-bg-success: #3aa716;--quarto-scss-export-valuebox-bg-info: rgba(153, 84, 187, 0.7019607843);--quarto-scss-export-valuebox-bg-warning: #fa6400;--quarto-scss-export-valuebox-bg-danger: rgba(255, 0, 57, 0.7019607843);--quarto-scss-export-valuebox-bg-light: #f8f9fa;--quarto-scss-export-valuebox-bg-dark: #343a40;--quarto-scss-export-mermaid-bg-color: #fff;--quarto-scss-export-mermaid-edge-color: #343a40;--quarto-scss-export-mermaid-node-fg-color: #343a40;--quarto-scss-export-mermaid-fg-color: #343a40;--quarto-scss-export-mermaid-fg-color--lighter: #4b545c;--quarto-scss-export-mermaid-fg-color--lightest: #626d78;--quarto-scss-export-mermaid-label-bg-color: #fff;--quarto-scss-export-mermaid-label-fg-color: #2780e3;--quarto-scss-export-mermaid-node-bg-color: rgba(39, 128, 227, 0.1);--quarto-scss-export-code-block-border-left-color: #dee2e6;--quarto-scss-export-callout-color-note: #2780e3;--quarto-scss-export-callout-color-tip: #3fb618;--quarto-scss-export-callout-color-important: #ff0039;--quarto-scss-export-callout-color-caution: #f0ad4e;--quarto-scss-export-callout-color-warning: #ff7518} \ No newline at end of file diff --git a/docs/site_libs/crosstalk-1.2.1/css/crosstalk.min.css b/docs/site_libs/crosstalk-1.2.1/css/crosstalk.min.css new file mode 100644 index 00000000..6b453828 --- /dev/null +++ b/docs/site_libs/crosstalk-1.2.1/css/crosstalk.min.css @@ -0,0 +1 @@ +.container-fluid.crosstalk-bscols{margin-left:-30px;margin-right:-30px;white-space:normal}body>.container-fluid.crosstalk-bscols{margin-left:auto;margin-right:auto}.crosstalk-input-checkboxgroup .crosstalk-options-group .crosstalk-options-column{display:inline-block;padding-right:12px;vertical-align:top}@media only screen and (max-width: 480px){.crosstalk-input-checkboxgroup .crosstalk-options-group .crosstalk-options-column{display:block;padding-right:inherit}}.crosstalk-input{margin-bottom:15px}.crosstalk-input .control-label{margin-bottom:0;vertical-align:middle}.crosstalk-input input[type="checkbox"]{margin:4px 0 0;margin-top:1px;line-height:normal}.crosstalk-input .checkbox{position:relative;display:block;margin-top:10px;margin-bottom:10px}.crosstalk-input .checkbox>label{padding-left:20px;margin-bottom:0;font-weight:400;cursor:pointer}.crosstalk-input .checkbox input[type="checkbox"],.crosstalk-input .checkbox-inline input[type="checkbox"]{position:absolute;margin-top:2px;margin-left:-20px}.crosstalk-input .checkbox+.checkbox{margin-top:-5px}.crosstalk-input .checkbox-inline{position:relative;display:inline-block;padding-left:20px;margin-bottom:0;font-weight:400;vertical-align:middle;cursor:pointer}.crosstalk-input .checkbox-inline+.checkbox-inline{margin-top:0;margin-left:10px} diff --git a/docs/site_libs/crosstalk-1.2.1/js/crosstalk.js b/docs/site_libs/crosstalk-1.2.1/js/crosstalk.js new file mode 100644 index 00000000..fd9eb53d --- /dev/null +++ b/docs/site_libs/crosstalk-1.2.1/js/crosstalk.js @@ -0,0 +1,1474 @@ +(function(){function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o b) { + return 1; + } +} + +/** + * @private + */ + +var FilterSet = function () { + function FilterSet() { + _classCallCheck(this, FilterSet); + + this.reset(); + } + + _createClass(FilterSet, [{ + key: "reset", + value: function reset() { + // Key: handle ID, Value: array of selected keys, or null + this._handles = {}; + // Key: key string, Value: count of handles that include it + this._keys = {}; + this._value = null; + this._activeHandles = 0; + } + }, { + key: "update", + value: function update(handleId, keys) { + if (keys !== null) { + keys = keys.slice(0); // clone before sorting + keys.sort(naturalComparator); + } + + var _diffSortedLists = (0, _util.diffSortedLists)(this._handles[handleId], keys), + added = _diffSortedLists.added, + removed = _diffSortedLists.removed; + + this._handles[handleId] = keys; + + for (var i = 0; i < added.length; i++) { + this._keys[added[i]] = (this._keys[added[i]] || 0) + 1; + } + for (var _i = 0; _i < removed.length; _i++) { + this._keys[removed[_i]]--; + } + + this._updateValue(keys); + } + + /** + * @param {string[]} keys Sorted array of strings that indicate + * a superset of possible keys. + * @private + */ + + }, { + key: "_updateValue", + value: function _updateValue() { + var keys = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : this._allKeys; + + var handleCount = Object.keys(this._handles).length; + if (handleCount === 0) { + this._value = null; + } else { + this._value = []; + for (var i = 0; i < keys.length; i++) { + var count = this._keys[keys[i]]; + if (count === handleCount) { + this._value.push(keys[i]); + } + } + } + } + }, { + key: "clear", + value: function clear(handleId) { + if (typeof this._handles[handleId] === "undefined") { + return; + } + + var keys = this._handles[handleId]; + if (!keys) { + keys = []; + } + + for (var i = 0; i < keys.length; i++) { + this._keys[keys[i]]--; + } + delete this._handles[handleId]; + + this._updateValue(); + } + }, { + key: "value", + get: function get() { + return this._value; + } + }, { + key: "_allKeys", + get: function get() { + var allKeys = Object.keys(this._keys); + allKeys.sort(naturalComparator); + return allKeys; + } + }]); + + return FilterSet; +}(); + +exports.default = FilterSet; + +},{"./util":11}],4:[function(require,module,exports){ +(function (global){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; + +exports.default = group; + +var _var2 = require("./var"); + +var _var3 = _interopRequireDefault(_var2); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +// Use a global so that multiple copies of crosstalk.js can be loaded and still +// have groups behave as singletons across all copies. +global.__crosstalk_groups = global.__crosstalk_groups || {}; +var groups = global.__crosstalk_groups; + +function group(groupName) { + if (groupName && typeof groupName === "string") { + if (!groups.hasOwnProperty(groupName)) { + groups[groupName] = new Group(groupName); + } + return groups[groupName]; + } else if ((typeof groupName === "undefined" ? "undefined" : _typeof(groupName)) === "object" && groupName._vars && groupName.var) { + // Appears to already be a group object + return groupName; + } else if (Array.isArray(groupName) && groupName.length == 1 && typeof groupName[0] === "string") { + return group(groupName[0]); + } else { + throw new Error("Invalid groupName argument"); + } +} + +var Group = function () { + function Group(name) { + _classCallCheck(this, Group); + + this.name = name; + this._vars = {}; + } + + _createClass(Group, [{ + key: "var", + value: function _var(name) { + if (!name || typeof name !== "string") { + throw new Error("Invalid var name"); + } + + if (!this._vars.hasOwnProperty(name)) this._vars[name] = new _var3.default(this, name); + return this._vars[name]; + } + }, { + key: "has", + value: function has(name) { + if (!name || typeof name !== "string") { + throw new Error("Invalid var name"); + } + + return this._vars.hasOwnProperty(name); + } + }]); + + return Group; +}(); + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./var":12}],5:[function(require,module,exports){ +(function (global){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _group = require("./group"); + +var _group2 = _interopRequireDefault(_group); + +var _selection = require("./selection"); + +var _filter = require("./filter"); + +var _input = require("./input"); + +require("./input_selectize"); + +require("./input_checkboxgroup"); + +require("./input_slider"); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +var defaultGroup = (0, _group2.default)("default"); + +function var_(name) { + return defaultGroup.var(name); +} + +function has(name) { + return defaultGroup.has(name); +} + +if (global.Shiny) { + global.Shiny.addCustomMessageHandler("update-client-value", function (message) { + if (typeof message.group === "string") { + (0, _group2.default)(message.group).var(message.name).set(message.value); + } else { + var_(message.name).set(message.value); + } + }); +} + +var crosstalk = { + group: _group2.default, + var: var_, + has: has, + SelectionHandle: _selection.SelectionHandle, + FilterHandle: _filter.FilterHandle, + bind: _input.bind +}; + +/** + * @namespace crosstalk + */ +exports.default = crosstalk; + +global.crosstalk = crosstalk; + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./filter":2,"./group":4,"./input":6,"./input_checkboxgroup":7,"./input_selectize":8,"./input_slider":9,"./selection":10}],6:[function(require,module,exports){ +(function (global){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.register = register; +exports.bind = bind; +var $ = global.jQuery; + +var bindings = {}; + +function register(reg) { + bindings[reg.className] = reg; + if (global.document && global.document.readyState !== "complete") { + $(function () { + bind(); + }); + } else if (global.document) { + setTimeout(bind, 100); + } +} + +function bind() { + Object.keys(bindings).forEach(function (className) { + var binding = bindings[className]; + $("." + binding.className).not(".crosstalk-input-bound").each(function (i, el) { + bindInstance(binding, el); + }); + }); +} + +// Escape jQuery identifier +function $escape(val) { + return val.replace(/([!"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, "\\$1"); +} + +function bindEl(el) { + var $el = $(el); + Object.keys(bindings).forEach(function (className) { + if ($el.hasClass(className) && !$el.hasClass("crosstalk-input-bound")) { + var binding = bindings[className]; + bindInstance(binding, el); + } + }); +} + +function bindInstance(binding, el) { + var jsonEl = $(el).find("script[type='application/json'][data-for='" + $escape(el.id) + "']"); + var data = JSON.parse(jsonEl[0].innerText); + + var instance = binding.factory(el, data); + $(el).data("crosstalk-instance", instance); + $(el).addClass("crosstalk-input-bound"); +} + +if (global.Shiny) { + var inputBinding = new global.Shiny.InputBinding(); + var _$ = global.jQuery; + _$.extend(inputBinding, { + find: function find(scope) { + return _$(scope).find(".crosstalk-input"); + }, + initialize: function initialize(el) { + if (!_$(el).hasClass("crosstalk-input-bound")) { + bindEl(el); + } + }, + getId: function getId(el) { + return el.id; + }, + getValue: function getValue(el) {}, + setValue: function setValue(el, value) {}, + receiveMessage: function receiveMessage(el, data) {}, + subscribe: function subscribe(el, callback) { + _$(el).data("crosstalk-instance").resume(); + }, + unsubscribe: function unsubscribe(el) { + _$(el).data("crosstalk-instance").suspend(); + } + }); + global.Shiny.inputBindings.register(inputBinding, "crosstalk.inputBinding"); +} + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{}],7:[function(require,module,exports){ +(function (global){ +"use strict"; + +var _input = require("./input"); + +var input = _interopRequireWildcard(_input); + +var _filter = require("./filter"); + +function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } } + +var $ = global.jQuery; + +input.register({ + className: "crosstalk-input-checkboxgroup", + + factory: function factory(el, data) { + /* + * map: {"groupA": ["keyA", "keyB", ...], ...} + * group: "ct-groupname" + */ + var ctHandle = new _filter.FilterHandle(data.group); + + var lastKnownKeys = void 0; + var $el = $(el); + $el.on("change", "input[type='checkbox']", function () { + var checked = $el.find("input[type='checkbox']:checked"); + if (checked.length === 0) { + lastKnownKeys = null; + ctHandle.clear(); + } else { + var keys = {}; + checked.each(function () { + data.map[this.value].forEach(function (key) { + keys[key] = true; + }); + }); + var keyArray = Object.keys(keys); + keyArray.sort(); + lastKnownKeys = keyArray; + ctHandle.set(keyArray); + } + }); + + return { + suspend: function suspend() { + ctHandle.clear(); + }, + resume: function resume() { + if (lastKnownKeys) ctHandle.set(lastKnownKeys); + } + }; + } +}); + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./filter":2,"./input":6}],8:[function(require,module,exports){ +(function (global){ +"use strict"; + +var _input = require("./input"); + +var input = _interopRequireWildcard(_input); + +var _util = require("./util"); + +var util = _interopRequireWildcard(_util); + +var _filter = require("./filter"); + +function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } } + +var $ = global.jQuery; + +input.register({ + className: "crosstalk-input-select", + + factory: function factory(el, data) { + /* + * items: {value: [...], label: [...]} + * map: {"groupA": ["keyA", "keyB", ...], ...} + * group: "ct-groupname" + */ + + var first = [{ value: "", label: "(All)" }]; + var items = util.dataframeToD3(data.items); + var opts = { + options: first.concat(items), + valueField: "value", + labelField: "label", + searchField: "label" + }; + + var select = $(el).find("select")[0]; + + var selectize = $(select).selectize(opts)[0].selectize; + + var ctHandle = new _filter.FilterHandle(data.group); + + var lastKnownKeys = void 0; + selectize.on("change", function () { + if (selectize.items.length === 0) { + lastKnownKeys = null; + ctHandle.clear(); + } else { + var keys = {}; + selectize.items.forEach(function (group) { + data.map[group].forEach(function (key) { + keys[key] = true; + }); + }); + var keyArray = Object.keys(keys); + keyArray.sort(); + lastKnownKeys = keyArray; + ctHandle.set(keyArray); + } + }); + + return { + suspend: function suspend() { + ctHandle.clear(); + }, + resume: function resume() { + if (lastKnownKeys) ctHandle.set(lastKnownKeys); + } + }; + } +}); + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./filter":2,"./input":6,"./util":11}],9:[function(require,module,exports){ +(function (global){ +"use strict"; + +var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }(); + +var _input = require("./input"); + +var input = _interopRequireWildcard(_input); + +var _filter = require("./filter"); + +function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } } + +var $ = global.jQuery; +var strftime = global.strftime; + +input.register({ + className: "crosstalk-input-slider", + + factory: function factory(el, data) { + /* + * map: {"groupA": ["keyA", "keyB", ...], ...} + * group: "ct-groupname" + */ + var ctHandle = new _filter.FilterHandle(data.group); + + var opts = {}; + var $el = $(el).find("input"); + var dataType = $el.data("data-type"); + var timeFormat = $el.data("time-format"); + var round = $el.data("round"); + var timeFormatter = void 0; + + // Set up formatting functions + if (dataType === "date") { + timeFormatter = strftime.utc(); + opts.prettify = function (num) { + return timeFormatter(timeFormat, new Date(num)); + }; + } else if (dataType === "datetime") { + var timezone = $el.data("timezone"); + if (timezone) timeFormatter = strftime.timezone(timezone);else timeFormatter = strftime; + + opts.prettify = function (num) { + return timeFormatter(timeFormat, new Date(num)); + }; + } else if (dataType === "number") { + if (typeof round !== "undefined") opts.prettify = function (num) { + var factor = Math.pow(10, round); + return Math.round(num * factor) / factor; + }; + } + + $el.ionRangeSlider(opts); + + function getValue() { + var result = $el.data("ionRangeSlider").result; + + // Function for converting numeric value from slider to appropriate type. + var convert = void 0; + var dataType = $el.data("data-type"); + if (dataType === "date") { + convert = function convert(val) { + return formatDateUTC(new Date(+val)); + }; + } else if (dataType === "datetime") { + convert = function convert(val) { + // Convert ms to s + return +val / 1000; + }; + } else { + convert = function convert(val) { + return +val; + }; + } + + if ($el.data("ionRangeSlider").options.type === "double") { + return [convert(result.from), convert(result.to)]; + } else { + return convert(result.from); + } + } + + var lastKnownKeys = null; + + $el.on("change.crosstalkSliderInput", function (event) { + if (!$el.data("updating") && !$el.data("animating")) { + var _getValue = getValue(), + _getValue2 = _slicedToArray(_getValue, 2), + from = _getValue2[0], + to = _getValue2[1]; + + var keys = []; + for (var i = 0; i < data.values.length; i++) { + var val = data.values[i]; + if (val >= from && val <= to) { + keys.push(data.keys[i]); + } + } + keys.sort(); + ctHandle.set(keys); + lastKnownKeys = keys; + } + }); + + // let $el = $(el); + // $el.on("change", "input[type="checkbox"]", function() { + // let checked = $el.find("input[type="checkbox"]:checked"); + // if (checked.length === 0) { + // ctHandle.clear(); + // } else { + // let keys = {}; + // checked.each(function() { + // data.map[this.value].forEach(function(key) { + // keys[key] = true; + // }); + // }); + // let keyArray = Object.keys(keys); + // keyArray.sort(); + // ctHandle.set(keyArray); + // } + // }); + + return { + suspend: function suspend() { + ctHandle.clear(); + }, + resume: function resume() { + if (lastKnownKeys) ctHandle.set(lastKnownKeys); + } + }; + } +}); + +// Convert a number to a string with leading zeros +function padZeros(n, digits) { + var str = n.toString(); + while (str.length < digits) { + str = "0" + str; + }return str; +} + +// Given a Date object, return a string in yyyy-mm-dd format, using the +// UTC date. This may be a day off from the date in the local time zone. +function formatDateUTC(date) { + if (date instanceof Date) { + return date.getUTCFullYear() + "-" + padZeros(date.getUTCMonth() + 1, 2) + "-" + padZeros(date.getUTCDate(), 2); + } else { + return null; + } +} + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./filter":2,"./input":6}],10:[function(require,module,exports){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.SelectionHandle = undefined; + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _events = require("./events"); + +var _events2 = _interopRequireDefault(_events); + +var _group = require("./group"); + +var _group2 = _interopRequireDefault(_group); + +var _util = require("./util"); + +var util = _interopRequireWildcard(_util); + +function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } } + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +/** + * Use this class to read and write (and listen for changes to) the selection + * for a Crosstalk group. This is intended to be used for linked brushing. + * + * If two (or more) `SelectionHandle` instances in the same webpage share the + * same group name, they will share the same state. Setting the selection using + * one `SelectionHandle` instance will result in the `value` property instantly + * changing across the others, and `"change"` event listeners on all instances + * (including the one that initiated the sending) will fire. + * + * @param {string} [group] - The name of the Crosstalk group, or if none, + * null or undefined (or any other falsy value). This can be changed later + * via the [SelectionHandle#setGroup](#setGroup) method. + * @param {Object} [extraInfo] - An object whose properties will be copied to + * the event object whenever an event is emitted. + */ +var SelectionHandle = exports.SelectionHandle = function () { + function SelectionHandle() { + var group = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : null; + var extraInfo = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : null; + + _classCallCheck(this, SelectionHandle); + + this._eventRelay = new _events2.default(); + this._emitter = new util.SubscriptionTracker(this._eventRelay); + + // Name of the group we're currently tracking, if any. Can change over time. + this._group = null; + // The Var we're currently tracking, if any. Can change over time. + this._var = null; + // The event handler subscription we currently have on var.on("change"). + this._varOnChangeSub = null; + + this._extraInfo = util.extend({ sender: this }, extraInfo); + + this.setGroup(group); + } + + /** + * Changes the Crosstalk group membership of this SelectionHandle. The group + * being switched away from (if any) will not have its selection value + * modified as a result of calling `setGroup`, even if this handle was the + * most recent handle to set the selection of the group. + * + * The group being switched to (if any) will also not have its selection value + * modified as a result of calling `setGroup`. If you want to set the + * selection value of the new group, call `set` explicitly. + * + * @param {string} group - The name of the Crosstalk group, or null (or + * undefined) to clear the group. + */ + + + _createClass(SelectionHandle, [{ + key: "setGroup", + value: function setGroup(group) { + var _this = this; + + // If group is unchanged, do nothing + if (this._group === group) return; + // Treat null, undefined, and other falsy values the same + if (!this._group && !group) return; + + if (this._var) { + this._var.off("change", this._varOnChangeSub); + this._var = null; + this._varOnChangeSub = null; + } + + this._group = group; + + if (group) { + this._var = (0, _group2.default)(group).var("selection"); + var sub = this._var.on("change", function (e) { + _this._eventRelay.trigger("change", e, _this); + }); + this._varOnChangeSub = sub; + } + } + + /** + * Retrieves the current selection for the group represented by this + * `SelectionHandle`. + * + * - If no selection is active, then this value will be falsy. + * - If a selection is active, but no data points are selected, then this + * value will be an empty array. + * - If a selection is active, and data points are selected, then the keys + * of the selected data points will be present in the array. + */ + + }, { + key: "_mergeExtraInfo", + + + /** + * Combines the given `extraInfo` (if any) with the handle's default + * `_extraInfo` (if any). + * @private + */ + value: function _mergeExtraInfo(extraInfo) { + // Important incidental effect: shallow clone is returned + return util.extend({}, this._extraInfo ? this._extraInfo : null, extraInfo ? extraInfo : null); + } + + /** + * Overwrites the current selection for the group, and raises the `"change"` + * event among all of the group's '`SelectionHandle` instances (including + * this one). + * + * @fires SelectionHandle#change + * @param {string[]} selectedKeys - Falsy, empty array, or array of keys (see + * {@link SelectionHandle#value}). + * @param {Object} [extraInfo] - Extra properties to be included on the event + * object that's passed to listeners (in addition to any options that were + * passed into the `SelectionHandle` constructor). + */ + + }, { + key: "set", + value: function set(selectedKeys, extraInfo) { + if (this._var) this._var.set(selectedKeys, this._mergeExtraInfo(extraInfo)); + } + + /** + * Overwrites the current selection for the group, and raises the `"change"` + * event among all of the group's '`SelectionHandle` instances (including + * this one). + * + * @fires SelectionHandle#change + * @param {Object} [extraInfo] - Extra properties to be included on the event + * object that's passed to listeners (in addition to any that were passed + * into the `SelectionHandle` constructor). + */ + + }, { + key: "clear", + value: function clear(extraInfo) { + if (this._var) this.set(void 0, this._mergeExtraInfo(extraInfo)); + } + + /** + * Subscribes to events on this `SelectionHandle`. + * + * @param {string} eventType - Indicates the type of events to listen to. + * Currently, only `"change"` is supported. + * @param {SelectionHandle~listener} listener - The callback function that + * will be invoked when the event occurs. + * @return {string} - A token to pass to {@link SelectionHandle#off} to cancel + * this subscription. + */ + + }, { + key: "on", + value: function on(eventType, listener) { + return this._emitter.on(eventType, listener); + } + + /** + * Cancels event subscriptions created by {@link SelectionHandle#on}. + * + * @param {string} eventType - The type of event to unsubscribe. + * @param {string|SelectionHandle~listener} listener - Either the callback + * function previously passed into {@link SelectionHandle#on}, or the + * string that was returned from {@link SelectionHandle#on}. + */ + + }, { + key: "off", + value: function off(eventType, listener) { + return this._emitter.off(eventType, listener); + } + + /** + * Shuts down the `SelectionHandle` object. + * + * Removes all event listeners that were added through this handle. + */ + + }, { + key: "close", + value: function close() { + this._emitter.removeAllListeners(); + this.setGroup(null); + } + }, { + key: "value", + get: function get() { + return this._var ? this._var.get() : null; + } + }]); + + return SelectionHandle; +}(); + +/** + * @callback SelectionHandle~listener + * @param {Object} event - An object containing details of the event. For + * `"change"` events, this includes the properties `value` (the new + * value of the selection, or `undefined` if no selection is active), + * `oldValue` (the previous value of the selection), and `sender` (the + * `SelectionHandle` instance that made the change). + */ + +/** + * @event SelectionHandle#change + * @type {object} + * @property {object} value - The new value of the selection, or `undefined` + * if no selection is active. + * @property {object} oldValue - The previous value of the selection. + * @property {SelectionHandle} sender - The `SelectionHandle` instance that + * changed the value. + */ + +},{"./events":1,"./group":4,"./util":11}],11:[function(require,module,exports){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; + +exports.extend = extend; +exports.checkSorted = checkSorted; +exports.diffSortedLists = diffSortedLists; +exports.dataframeToD3 = dataframeToD3; + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +function extend(target) { + for (var _len = arguments.length, sources = Array(_len > 1 ? _len - 1 : 0), _key = 1; _key < _len; _key++) { + sources[_key - 1] = arguments[_key]; + } + + for (var i = 0; i < sources.length; i++) { + var src = sources[i]; + if (typeof src === "undefined" || src === null) continue; + + for (var key in src) { + if (src.hasOwnProperty(key)) { + target[key] = src[key]; + } + } + } + return target; +} + +function checkSorted(list) { + for (var i = 1; i < list.length; i++) { + if (list[i] <= list[i - 1]) { + throw new Error("List is not sorted or contains duplicate"); + } + } +} + +function diffSortedLists(a, b) { + var i_a = 0; + var i_b = 0; + + if (!a) a = []; + if (!b) b = []; + + var a_only = []; + var b_only = []; + + checkSorted(a); + checkSorted(b); + + while (i_a < a.length && i_b < b.length) { + if (a[i_a] === b[i_b]) { + i_a++; + i_b++; + } else if (a[i_a] < b[i_b]) { + a_only.push(a[i_a++]); + } else { + b_only.push(b[i_b++]); + } + } + + if (i_a < a.length) a_only = a_only.concat(a.slice(i_a)); + if (i_b < b.length) b_only = b_only.concat(b.slice(i_b)); + return { + removed: a_only, + added: b_only + }; +} + +// Convert from wide: { colA: [1,2,3], colB: [4,5,6], ... } +// to long: [ {colA: 1, colB: 4}, {colA: 2, colB: 5}, ... ] +function dataframeToD3(df) { + var names = []; + var length = void 0; + for (var name in df) { + if (df.hasOwnProperty(name)) names.push(name); + if (_typeof(df[name]) !== "object" || typeof df[name].length === "undefined") { + throw new Error("All fields must be arrays"); + } else if (typeof length !== "undefined" && length !== df[name].length) { + throw new Error("All fields must be arrays of the same length"); + } + length = df[name].length; + } + var results = []; + var item = void 0; + for (var row = 0; row < length; row++) { + item = {}; + for (var col = 0; col < names.length; col++) { + item[names[col]] = df[names[col]][row]; + } + results.push(item); + } + return results; +} + +/** + * Keeps track of all event listener additions/removals and lets all active + * listeners be removed with a single operation. + * + * @private + */ + +var SubscriptionTracker = exports.SubscriptionTracker = function () { + function SubscriptionTracker(emitter) { + _classCallCheck(this, SubscriptionTracker); + + this._emitter = emitter; + this._subs = {}; + } + + _createClass(SubscriptionTracker, [{ + key: "on", + value: function on(eventType, listener) { + var sub = this._emitter.on(eventType, listener); + this._subs[sub] = eventType; + return sub; + } + }, { + key: "off", + value: function off(eventType, listener) { + var sub = this._emitter.off(eventType, listener); + if (sub) { + delete this._subs[sub]; + } + return sub; + } + }, { + key: "removeAllListeners", + value: function removeAllListeners() { + var _this = this; + + var current_subs = this._subs; + this._subs = {}; + Object.keys(current_subs).forEach(function (sub) { + _this._emitter.off(current_subs[sub], sub); + }); + } + }]); + + return SubscriptionTracker; +}(); + +},{}],12:[function(require,module,exports){ +(function (global){ +"use strict"; + +Object.defineProperty(exports, "__esModule", { + value: true +}); + +var _typeof = typeof Symbol === "function" && typeof Symbol.iterator === "symbol" ? function (obj) { return typeof obj; } : function (obj) { return obj && typeof Symbol === "function" && obj.constructor === Symbol && obj !== Symbol.prototype ? "symbol" : typeof obj; }; + +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + +var _events = require("./events"); + +var _events2 = _interopRequireDefault(_events); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +var Var = function () { + function Var(group, name, /*optional*/value) { + _classCallCheck(this, Var); + + this._group = group; + this._name = name; + this._value = value; + this._events = new _events2.default(); + } + + _createClass(Var, [{ + key: "get", + value: function get() { + return this._value; + } + }, { + key: "set", + value: function set(value, /*optional*/event) { + if (this._value === value) { + // Do nothing; the value hasn't changed + return; + } + var oldValue = this._value; + this._value = value; + // Alert JavaScript listeners that the value has changed + var evt = {}; + if (event && (typeof event === "undefined" ? "undefined" : _typeof(event)) === "object") { + for (var k in event) { + if (event.hasOwnProperty(k)) evt[k] = event[k]; + } + } + evt.oldValue = oldValue; + evt.value = value; + this._events.trigger("change", evt, this); + + // TODO: Make this extensible, to let arbitrary back-ends know that + // something has changed + if (global.Shiny && global.Shiny.onInputChange) { + global.Shiny.onInputChange(".clientValue-" + (this._group.name !== null ? this._group.name + "-" : "") + this._name, typeof value === "undefined" ? null : value); + } + } + }, { + key: "on", + value: function on(eventType, listener) { + return this._events.on(eventType, listener); + } + }, { + key: "off", + value: function off(eventType, listener) { + return this._events.off(eventType, listener); + } + }]); + + return Var; +}(); + +exports.default = Var; + +}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {}) + +},{"./events":1}]},{},[5]) +//# sourceMappingURL=crosstalk.js.map diff --git a/docs/site_libs/crosstalk-1.2.1/js/crosstalk.js.map b/docs/site_libs/crosstalk-1.2.1/js/crosstalk.js.map new file mode 100644 index 00000000..cff94f08 --- /dev/null +++ b/docs/site_libs/crosstalk-1.2.1/js/crosstalk.js.map @@ -0,0 +1,37 @@ +{ + "version": 3, + "sources": [ + "node_modules/browser-pack/_prelude.js", + "javascript/src/events.js", + "javascript/src/filter.js", + "javascript/src/filterset.js", + "javascript/src/group.js", + "javascript/src/index.js", + "javascript/src/input.js", + "javascript/src/input_checkboxgroup.js", + "javascript/src/input_selectize.js", + "javascript/src/input_slider.js", + "javascript/src/selection.js", + "javascript/src/util.js", + "javascript/src/var.js" + ], + "names": [], + "mappings": "AAAA;;;;;;;;;;;ICAqB,M;AACnB,oBAAc;AAAA;;AACZ,SAAK,MAAL,GAAc,EAAd;AACA,SAAK,IAAL,GAAY,CAAZ;AACD;;;;uBAEE,S,EAAW,Q,EAAU;AACtB,UAAI,OAAO,KAAK,MAAL,CAAY,SAAZ,CAAX;AACA,UAAI,CAAC,IAAL,EAAW;AACT,eAAO,KAAK,MAAL,CAAY,SAAZ,IAAyB,EAAhC;AACD;AACD,UAAI,MAAM,QAAS,KAAK,IAAL,EAAnB;AACA,WAAK,GAAL,IAAY,QAAZ;AACA,aAAO,GAAP;AACD;;AAED;;;;wBACI,S,EAAW,Q,EAAU;AACvB,UAAI,OAAO,KAAK,MAAL,CAAY,SAAZ,CAAX;AACA,UAAI,OAAO,QAAP,KAAqB,UAAzB,EAAqC;AACnC,aAAK,IAAI,GAAT,IAAgB,IAAhB,EAAsB;AACpB,cAAI,KAAK,cAAL,CAAoB,GAApB,CAAJ,EAA8B;AAC5B,gBAAI,KAAK,GAAL,MAAc,QAAlB,EAA4B;AAC1B,qBAAO,KAAK,GAAL,CAAP;AACA,qBAAO,GAAP;AACD;AACF;AACF;AACD,eAAO,KAAP;AACD,OAVD,MAUO,IAAI,OAAO,QAAP,KAAqB,QAAzB,EAAmC;AACxC,YAAI,QAAQ,KAAK,QAAL,CAAZ,EAA4B;AAC1B,iBAAO,KAAK,QAAL,CAAP;AACA,iBAAO,QAAP;AACD;AACD,eAAO,KAAP;AACD,OANM,MAMA;AACL,cAAM,IAAI,KAAJ,CAAU,8BAAV,CAAN;AACD;AACF;;;4BAEO,S,EAAW,G,EAAK,O,EAAS;AAC/B,UAAI,OAAO,KAAK,MAAL,CAAY,SAAZ,CAAX;AACA,WAAK,IAAI,GAAT,IAAgB,IAAhB,EAAsB;AACpB,YAAI,KAAK,cAAL,CAAoB,GAApB,CAAJ,EAA8B;AAC5B,eAAK,GAAL,EAAU,IAAV,CAAe,OAAf,EAAwB,GAAxB;AACD;AACF;AACF;;;;;;kBA/CkB,M;;;;;;;;;;;;ACArB;;;;AACA;;;;AACA;;;;AACA;;IAAY,I;;;;;;;;AAEZ,SAAS,YAAT,CAAsB,KAAtB,EAA6B;AAC3B,MAAI,QAAQ,MAAM,GAAN,CAAU,WAAV,CAAZ;AACA,MAAI,SAAS,MAAM,GAAN,EAAb;AACA,MAAI,CAAC,MAAL,EAAa;AACX,aAAS,yBAAT;AACA,UAAM,GAAN,CAAU,MAAV;AACD;AACD,SAAO,MAAP;AACD;;AAED,IAAI,KAAK,CAAT;AACA,SAAS,MAAT,GAAkB;AAChB,SAAO,IAAP;AACD;;AAED;;;;;;;;;;;;;;;;;;;;;;;;;IAwBa,Y,WAAA,Y;AACX,wBAAY,KAAZ,EAAmB,SAAnB,EAA8B;AAAA;;AAC5B,SAAK,WAAL,GAAmB,sBAAnB;AACA,SAAK,QAAL,GAAgB,IAAI,KAAK,mBAAT,CAA6B,KAAK,WAAlC,CAAhB;;AAEA;AACA,SAAK,MAAL,GAAc,IAAd;AACA;AACA,SAAK,UAAL,GAAkB,IAAlB;AACA;AACA,SAAK,UAAL,GAAkB,IAAlB;AACA;AACA,SAAK,eAAL,GAAuB,IAAvB;;AAEA,SAAK,UAAL,GAAkB,KAAK,MAAL,CAAY,EAAE,QAAQ,IAAV,EAAZ,EAA8B,SAA9B,CAAlB;;AAEA,SAAK,GAAL,GAAW,WAAW,QAAtB;;AAEA,SAAK,QAAL,CAAc,KAAd;AACD;;AAED;;;;;;;;;;;;;;6BAUS,K,EAAO;AAAA;;AACd;AACA,UAAI,KAAK,MAAL,KAAgB,KAApB,EACE;AACF;AACA,UAAI,CAAC,KAAK,MAAN,IAAgB,CAAC,KAArB,EACE;;AAEF,UAAI,KAAK,UAAT,EAAqB;AACnB,aAAK,UAAL,CAAgB,GAAhB,CAAoB,QAApB,EAA8B,KAAK,eAAnC;AACA,aAAK,KAAL;AACA,aAAK,eAAL,GAAuB,IAAvB;AACA,aAAK,UAAL,GAAkB,IAAlB;AACA,aAAK,UAAL,GAAkB,IAAlB;AACD;;AAED,WAAK,MAAL,GAAc,KAAd;;AAEA,UAAI,KAAJ,EAAW;AACT,gBAAQ,qBAAI,KAAJ,CAAR;AACA,aAAK,UAAL,GAAkB,aAAa,KAAb,CAAlB;AACA,aAAK,UAAL,GAAkB,qBAAI,KAAJ,EAAW,GAAX,CAAe,QAAf,CAAlB;AACA,YAAI,MAAM,KAAK,UAAL,CAAgB,EAAhB,CAAmB,QAAnB,EAA6B,UAAC,CAAD,EAAO;AAC5C,gBAAK,WAAL,CAAiB,OAAjB,CAAyB,QAAzB,EAAmC,CAAnC;AACD,SAFS,CAAV;AAGA,aAAK,eAAL,GAAuB,GAAvB;AACD;AACF;;AAED;;;;;;;;oCAKgB,S,EAAW;AACzB,aAAO,KAAK,MAAL,CAAY,EAAZ,EACL,KAAK,UAAL,GAAkB,KAAK,UAAvB,GAAoC,IAD/B,EAEL,YAAY,SAAZ,GAAwB,IAFnB,CAAP;AAGD;;AAED;;;;;;;4BAIQ;AACN,WAAK,QAAL,CAAc,kBAAd;AACA,WAAK,KAAL;AACA,WAAK,QAAL,CAAc,IAAd;AACD;;AAED;;;;;;;;;;;;0BASM,S,EAAW;AACf,UAAI,CAAC,KAAK,UAAV,EACE;AACF,WAAK,UAAL,CAAgB,KAAhB,CAAsB,KAAK,GAA3B;AACA,WAAK,SAAL,CAAe,SAAf;AACD;;AAED;;;;;;;;;;;;;;;;;;;;wBAiBI,I,EAAM,S,EAAW;AACnB,UAAI,CAAC,KAAK,UAAV,EACE;AACF,WAAK,UAAL,CAAgB,MAAhB,CAAuB,KAAK,GAA5B,EAAiC,IAAjC;AACA,WAAK,SAAL,CAAe,SAAf;AACD;;AAED;;;;;;;;;;AASA;;;;;;;;;;uBAUG,S,EAAW,Q,EAAU;AACtB,aAAO,KAAK,QAAL,CAAc,EAAd,CAAiB,SAAjB,EAA4B,QAA5B,CAAP;AACD;;AAED;;;;;;;;;;;wBAQI,S,EAAW,Q,EAAU;AACvB,aAAO,KAAK,QAAL,CAAc,GAAd,CAAkB,SAAlB,EAA6B,QAA7B,CAAP;AACD;;;8BAES,S,EAAW;AACnB,UAAI,CAAC,KAAK,UAAV,EACE;AACF,WAAK,UAAL,CAAgB,GAAhB,CAAoB,KAAK,UAAL,CAAgB,KAApC,EAA2C,KAAK,eAAL,CAAqB,SAArB,CAA3C;AACD;;AAED;;;;;;;;;;;wBApCmB;AACjB,aAAO,KAAK,UAAL,GAAkB,KAAK,UAAL,CAAgB,KAAlC,GAA0C,IAAjD;AACD;;;;;;AA6CH;;;;;;;;;;;;;;;;;;;ACzNA;;;;AAEA,SAAS,iBAAT,CAA2B,CAA3B,EAA8B,CAA9B,EAAiC;AAC/B,MAAI,MAAM,CAAV,EAAa;AACX,WAAO,CAAP;AACD,GAFD,MAEO,IAAI,IAAI,CAAR,EAAW;AAChB,WAAO,CAAC,CAAR;AACD,GAFM,MAEA,IAAI,IAAI,CAAR,EAAW;AAChB,WAAO,CAAP;AACD;AACF;;AAED;;;;IAGqB,S;AACnB,uBAAc;AAAA;;AACZ,SAAK,KAAL;AACD;;;;4BAEO;AACN;AACA,WAAK,QAAL,GAAgB,EAAhB;AACA;AACA,WAAK,KAAL,GAAa,EAAb;AACA,WAAK,MAAL,GAAc,IAAd;AACA,WAAK,cAAL,GAAsB,CAAtB;AACD;;;2BAMM,Q,EAAU,I,EAAM;AACrB,UAAI,SAAS,IAAb,EAAmB;AACjB,eAAO,KAAK,KAAL,CAAW,CAAX,CAAP,CADiB,CACK;AACtB,aAAK,IAAL,CAAU,iBAAV;AACD;;AAJoB,6BAME,2BAAgB,KAAK,QAAL,CAAc,QAAd,CAAhB,EAAyC,IAAzC,CANF;AAAA,UAMhB,KANgB,oBAMhB,KANgB;AAAA,UAMT,OANS,oBAMT,OANS;;AAOrB,WAAK,QAAL,CAAc,QAAd,IAA0B,IAA1B;;AAEA,WAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,MAAM,MAA1B,EAAkC,GAAlC,EAAuC;AACrC,aAAK,KAAL,CAAW,MAAM,CAAN,CAAX,IAAuB,CAAC,KAAK,KAAL,CAAW,MAAM,CAAN,CAAX,KAAwB,CAAzB,IAA8B,CAArD;AACD;AACD,WAAK,IAAI,KAAI,CAAb,EAAgB,KAAI,QAAQ,MAA5B,EAAoC,IAApC,EAAyC;AACvC,aAAK,KAAL,CAAW,QAAQ,EAAR,CAAX;AACD;;AAED,WAAK,YAAL,CAAkB,IAAlB;AACD;;AAED;;;;;;;;mCAKmC;AAAA,UAAtB,IAAsB,uEAAf,KAAK,QAAU;;AACjC,UAAI,cAAc,OAAO,IAAP,CAAY,KAAK,QAAjB,EAA2B,MAA7C;AACA,UAAI,gBAAgB,CAApB,EAAuB;AACrB,aAAK,MAAL,GAAc,IAAd;AACD,OAFD,MAEO;AACL,aAAK,MAAL,GAAc,EAAd;AACA,aAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,KAAK,MAAzB,EAAiC,GAAjC,EAAsC;AACpC,cAAI,QAAQ,KAAK,KAAL,CAAW,KAAK,CAAL,CAAX,CAAZ;AACA,cAAI,UAAU,WAAd,EAA2B;AACzB,iBAAK,MAAL,CAAY,IAAZ,CAAiB,KAAK,CAAL,CAAjB;AACD;AACF;AACF;AACF;;;0BAEK,Q,EAAU;AACd,UAAI,OAAO,KAAK,QAAL,CAAc,QAAd,CAAP,KAAoC,WAAxC,EAAqD;AACnD;AACD;;AAED,UAAI,OAAO,KAAK,QAAL,CAAc,QAAd,CAAX;AACA,UAAI,CAAC,IAAL,EAAW;AACT,eAAO,EAAP;AACD;;AAED,WAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,KAAK,MAAzB,EAAiC,GAAjC,EAAsC;AACpC,aAAK,KAAL,CAAW,KAAK,CAAL,CAAX;AACD;AACD,aAAO,KAAK,QAAL,CAAc,QAAd,CAAP;;AAEA,WAAK,YAAL;AACD;;;wBA3DW;AACV,aAAO,KAAK,MAAZ;AACD;;;wBA2Dc;AACb,UAAI,UAAU,OAAO,IAAP,CAAY,KAAK,KAAjB,CAAd;AACA,cAAQ,IAAR,CAAa,iBAAb;AACA,aAAO,OAAP;AACD;;;;;;kBA/EkB,S;;;;;;;;;;;;;;kBCRG,K;;AAPxB;;;;;;;;AAEA;AACA;AACA,OAAO,kBAAP,GAA4B,OAAO,kBAAP,IAA6B,EAAzD;AACA,IAAI,SAAS,OAAO,kBAApB;;AAEe,SAAS,KAAT,CAAe,SAAf,EAA0B;AACvC,MAAI,aAAa,OAAO,SAAP,KAAsB,QAAvC,EAAiD;AAC/C,QAAI,CAAC,OAAO,cAAP,CAAsB,SAAtB,CAAL,EAAuC;AACrC,aAAO,SAAP,IAAoB,IAAI,KAAJ,CAAU,SAAV,CAApB;AACD;AACD,WAAO,OAAO,SAAP,CAAP;AACD,GALD,MAKO,IAAI,QAAO,SAAP,yCAAO,SAAP,OAAsB,QAAtB,IAAkC,UAAU,KAA5C,IAAqD,UAAU,GAAnE,EAAwE;AAC7E;AACA,WAAO,SAAP;AACD,GAHM,MAGA,IAAI,MAAM,OAAN,CAAc,SAAd,KACP,UAAU,MAAV,IAAoB,CADb,IAEP,OAAO,UAAU,CAAV,CAAP,KAAyB,QAFtB,EAEgC;AACrC,WAAO,MAAM,UAAU,CAAV,CAAN,CAAP;AACD,GAJM,MAIA;AACL,UAAM,IAAI,KAAJ,CAAU,4BAAV,CAAN;AACD;AACF;;IAEK,K;AACJ,iBAAY,IAAZ,EAAkB;AAAA;;AAChB,SAAK,IAAL,GAAY,IAAZ;AACA,SAAK,KAAL,GAAa,EAAb;AACD;;;;yBAEG,I,EAAM;AACR,UAAI,CAAC,IAAD,IAAS,OAAO,IAAP,KAAiB,QAA9B,EAAwC;AACtC,cAAM,IAAI,KAAJ,CAAU,kBAAV,CAAN;AACD;;AAED,UAAI,CAAC,KAAK,KAAL,CAAW,cAAX,CAA0B,IAA1B,CAAL,EACE,KAAK,KAAL,CAAW,IAAX,IAAmB,kBAAQ,IAAR,EAAc,IAAd,CAAnB;AACF,aAAO,KAAK,KAAL,CAAW,IAAX,CAAP;AACD;;;wBAEG,I,EAAM;AACR,UAAI,CAAC,IAAD,IAAS,OAAO,IAAP,KAAiB,QAA9B,EAAwC;AACtC,cAAM,IAAI,KAAJ,CAAU,kBAAV,CAAN;AACD;;AAED,aAAO,KAAK,KAAL,CAAW,cAAX,CAA0B,IAA1B,CAAP;AACD;;;;;;;;;;;;;;;;AC/CH;;;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAEA,IAAM,eAAe,qBAAM,SAAN,CAArB;;AAEA,SAAS,IAAT,CAAc,IAAd,EAAoB;AAClB,SAAO,aAAa,GAAb,CAAiB,IAAjB,CAAP;AACD;;AAED,SAAS,GAAT,CAAa,IAAb,EAAmB;AACjB,SAAO,aAAa,GAAb,CAAiB,IAAjB,CAAP;AACD;;AAED,IAAI,OAAO,KAAX,EAAkB;AAChB,SAAO,KAAP,CAAa,uBAAb,CAAqC,qBAArC,EAA4D,UAAS,OAAT,EAAkB;AAC5E,QAAI,OAAO,QAAQ,KAAf,KAA0B,QAA9B,EAAwC;AACtC,2BAAM,QAAQ,KAAd,EAAqB,GAArB,CAAyB,QAAQ,IAAjC,EAAuC,GAAvC,CAA2C,QAAQ,KAAnD;AACD,KAFD,MAEO;AACL,WAAK,QAAQ,IAAb,EAAmB,GAAnB,CAAuB,QAAQ,KAA/B;AACD;AACF,GAND;AAOD;;AAED,IAAM,YAAY;AAChB,wBADgB;AAEhB,OAAK,IAFW;AAGhB,OAAK,GAHW;AAIhB,6CAJgB;AAKhB,oCALgB;AAMhB;AANgB,CAAlB;;AASA;;;kBAGe,S;;AACf,OAAO,SAAP,GAAmB,SAAnB;;;;;;;;;;;QCrCgB,Q,GAAA,Q;QAWA,I,GAAA,I;AAfhB,IAAI,IAAI,OAAO,MAAf;;AAEA,IAAI,WAAW,EAAf;;AAEO,SAAS,QAAT,CAAkB,GAAlB,EAAuB;AAC5B,WAAS,IAAI,SAAb,IAA0B,GAA1B;AACA,MAAI,OAAO,QAAP,IAAmB,OAAO,QAAP,CAAgB,UAAhB,KAA+B,UAAtD,EAAkE;AAChE,MAAE,YAAM;AACN;AACD,KAFD;AAGD,GAJD,MAIO,IAAI,OAAO,QAAX,EAAqB;AAC1B,eAAW,IAAX,EAAiB,GAAjB;AACD;AACF;;AAEM,SAAS,IAAT,GAAgB;AACrB,SAAO,IAAP,CAAY,QAAZ,EAAsB,OAAtB,CAA8B,UAAS,SAAT,EAAoB;AAChD,QAAI,UAAU,SAAS,SAAT,CAAd;AACA,MAAE,MAAM,QAAQ,SAAhB,EAA2B,GAA3B,CAA+B,wBAA/B,EAAyD,IAAzD,CAA8D,UAAS,CAAT,EAAY,EAAZ,EAAgB;AAC5E,mBAAa,OAAb,EAAsB,EAAtB;AACD,KAFD;AAGD,GALD;AAMD;;AAED;AACA,SAAS,OAAT,CAAiB,GAAjB,EAAsB;AACpB,SAAO,IAAI,OAAJ,CAAY,uCAAZ,EAAqD,MAArD,CAAP;AACD;;AAED,SAAS,MAAT,CAAgB,EAAhB,EAAoB;AAClB,MAAI,MAAM,EAAE,EAAF,CAAV;AACA,SAAO,IAAP,CAAY,QAAZ,EAAsB,OAAtB,CAA8B,UAAS,SAAT,EAAoB;AAChD,QAAI,IAAI,QAAJ,CAAa,SAAb,KAA2B,CAAC,IAAI,QAAJ,CAAa,uBAAb,CAAhC,EAAuE;AACrE,UAAI,UAAU,SAAS,SAAT,CAAd;AACA,mBAAa,OAAb,EAAsB,EAAtB;AACD;AACF,GALD;AAMD;;AAED,SAAS,YAAT,CAAsB,OAAtB,EAA+B,EAA/B,EAAmC;AACjC,MAAI,SAAS,EAAE,EAAF,EAAM,IAAN,CAAW,+CAA+C,QAAQ,GAAG,EAAX,CAA/C,GAAgE,IAA3E,CAAb;AACA,MAAI,OAAO,KAAK,KAAL,CAAW,OAAO,CAAP,EAAU,SAArB,CAAX;;AAEA,MAAI,WAAW,QAAQ,OAAR,CAAgB,EAAhB,EAAoB,IAApB,CAAf;AACA,IAAE,EAAF,EAAM,IAAN,CAAW,oBAAX,EAAiC,QAAjC;AACA,IAAE,EAAF,EAAM,QAAN,CAAe,uBAAf;AACD;;AAED,IAAI,OAAO,KAAX,EAAkB;AAChB,MAAI,eAAe,IAAI,OAAO,KAAP,CAAa,YAAjB,EAAnB;AACA,MAAI,KAAI,OAAO,MAAf;AACA,KAAE,MAAF,CAAS,YAAT,EAAuB;AACrB,UAAM,cAAS,KAAT,EAAgB;AACpB,aAAO,GAAE,KAAF,EAAS,IAAT,CAAc,kBAAd,CAAP;AACD,KAHoB;AAIrB,gBAAY,oBAAS,EAAT,EAAa;AACvB,UAAI,CAAC,GAAE,EAAF,EAAM,QAAN,CAAe,uBAAf,CAAL,EAA8C;AAC5C,eAAO,EAAP;AACD;AACF,KARoB;AASrB,WAAO,eAAS,EAAT,EAAa;AAClB,aAAO,GAAG,EAAV;AACD,KAXoB;AAYrB,cAAU,kBAAS,EAAT,EAAa,CAEtB,CAdoB;AAerB,cAAU,kBAAS,EAAT,EAAa,KAAb,EAAoB,CAE7B,CAjBoB;AAkBrB,oBAAgB,wBAAS,EAAT,EAAa,IAAb,EAAmB,CAElC,CApBoB;AAqBrB,eAAW,mBAAS,EAAT,EAAa,QAAb,EAAuB;AAChC,SAAE,EAAF,EAAM,IAAN,CAAW,oBAAX,EAAiC,MAAjC;AACD,KAvBoB;AAwBrB,iBAAa,qBAAS,EAAT,EAAa;AACxB,SAAE,EAAF,EAAM,IAAN,CAAW,oBAAX,EAAiC,OAAjC;AACD;AA1BoB,GAAvB;AA4BA,SAAO,KAAP,CAAa,aAAb,CAA2B,QAA3B,CAAoC,YAApC,EAAkD,wBAAlD;AACD;;;;;;;;AChFD;;IAAY,K;;AACZ;;;;AAEA,IAAI,IAAI,OAAO,MAAf;;AAEA,MAAM,QAAN,CAAe;AACb,aAAW,+BADE;;AAGb,WAAS,iBAAS,EAAT,EAAa,IAAb,EAAmB;AAC1B;;;;AAIA,QAAI,WAAW,yBAAiB,KAAK,KAAtB,CAAf;;AAEA,QAAI,sBAAJ;AACA,QAAI,MAAM,EAAE,EAAF,CAAV;AACA,QAAI,EAAJ,CAAO,QAAP,EAAiB,wBAAjB,EAA2C,YAAW;AACpD,UAAI,UAAU,IAAI,IAAJ,CAAS,gCAAT,CAAd;AACA,UAAI,QAAQ,MAAR,KAAmB,CAAvB,EAA0B;AACxB,wBAAgB,IAAhB;AACA,iBAAS,KAAT;AACD,OAHD,MAGO;AACL,YAAI,OAAO,EAAX;AACA,gBAAQ,IAAR,CAAa,YAAW;AACtB,eAAK,GAAL,CAAS,KAAK,KAAd,EAAqB,OAArB,CAA6B,UAAS,GAAT,EAAc;AACzC,iBAAK,GAAL,IAAY,IAAZ;AACD,WAFD;AAGD,SAJD;AAKA,YAAI,WAAW,OAAO,IAAP,CAAY,IAAZ,CAAf;AACA,iBAAS,IAAT;AACA,wBAAgB,QAAhB;AACA,iBAAS,GAAT,CAAa,QAAb;AACD;AACF,KAjBD;;AAmBA,WAAO;AACL,eAAS,mBAAW;AAClB,iBAAS,KAAT;AACD,OAHI;AAIL,cAAQ,kBAAW;AACjB,YAAI,aAAJ,EACE,SAAS,GAAT,CAAa,aAAb;AACH;AAPI,KAAP;AASD;AAxCY,CAAf;;;;;;;;ACLA;;IAAY,K;;AACZ;;IAAY,I;;AACZ;;;;AAEA,IAAI,IAAI,OAAO,MAAf;;AAEA,MAAM,QAAN,CAAe;AACb,aAAW,wBADE;;AAGb,WAAS,iBAAS,EAAT,EAAa,IAAb,EAAmB;AAC1B;;;;;;AAMA,QAAI,QAAQ,CAAC,EAAC,OAAO,EAAR,EAAY,OAAO,OAAnB,EAAD,CAAZ;AACA,QAAI,QAAQ,KAAK,aAAL,CAAmB,KAAK,KAAxB,CAAZ;AACA,QAAI,OAAO;AACT,eAAS,MAAM,MAAN,CAAa,KAAb,CADA;AAET,kBAAY,OAFH;AAGT,kBAAY,OAHH;AAIT,mBAAa;AAJJ,KAAX;;AAOA,QAAI,SAAS,EAAE,EAAF,EAAM,IAAN,CAAW,QAAX,EAAqB,CAArB,CAAb;;AAEA,QAAI,YAAY,EAAE,MAAF,EAAU,SAAV,CAAoB,IAApB,EAA0B,CAA1B,EAA6B,SAA7C;;AAEA,QAAI,WAAW,yBAAiB,KAAK,KAAtB,CAAf;;AAEA,QAAI,sBAAJ;AACA,cAAU,EAAV,CAAa,QAAb,EAAuB,YAAW;AAChC,UAAI,UAAU,KAAV,CAAgB,MAAhB,KAA2B,CAA/B,EAAkC;AAChC,wBAAgB,IAAhB;AACA,iBAAS,KAAT;AACD,OAHD,MAGO;AACL,YAAI,OAAO,EAAX;AACA,kBAAU,KAAV,CAAgB,OAAhB,CAAwB,UAAS,KAAT,EAAgB;AACtC,eAAK,GAAL,CAAS,KAAT,EAAgB,OAAhB,CAAwB,UAAS,GAAT,EAAc;AACpC,iBAAK,GAAL,IAAY,IAAZ;AACD,WAFD;AAGD,SAJD;AAKA,YAAI,WAAW,OAAO,IAAP,CAAY,IAAZ,CAAf;AACA,iBAAS,IAAT;AACA,wBAAgB,QAAhB;AACA,iBAAS,GAAT,CAAa,QAAb;AACD;AACF,KAhBD;;AAkBA,WAAO;AACL,eAAS,mBAAW;AAClB,iBAAS,KAAT;AACD,OAHI;AAIL,cAAQ,kBAAW;AACjB,YAAI,aAAJ,EACE,SAAS,GAAT,CAAa,aAAb;AACH;AAPI,KAAP;AASD;AArDY,CAAf;;;;;;;;;;ACNA;;IAAY,K;;AACZ;;;;AAEA,IAAI,IAAI,OAAO,MAAf;AACA,IAAI,WAAW,OAAO,QAAtB;;AAEA,MAAM,QAAN,CAAe;AACb,aAAW,wBADE;;AAGb,WAAS,iBAAS,EAAT,EAAa,IAAb,EAAmB;AAC1B;;;;AAIA,QAAI,WAAW,yBAAiB,KAAK,KAAtB,CAAf;;AAEA,QAAI,OAAO,EAAX;AACA,QAAI,MAAM,EAAE,EAAF,EAAM,IAAN,CAAW,OAAX,CAAV;AACA,QAAI,WAAW,IAAI,IAAJ,CAAS,WAAT,CAAf;AACA,QAAI,aAAa,IAAI,IAAJ,CAAS,aAAT,CAAjB;AACA,QAAI,QAAQ,IAAI,IAAJ,CAAS,OAAT,CAAZ;AACA,QAAI,sBAAJ;;AAEA;AACA,QAAI,aAAa,MAAjB,EAAyB;AACvB,sBAAgB,SAAS,GAAT,EAAhB;AACA,WAAK,QAAL,GAAgB,UAAS,GAAT,EAAc;AAC5B,eAAO,cAAc,UAAd,EAA0B,IAAI,IAAJ,CAAS,GAAT,CAA1B,CAAP;AACD,OAFD;AAID,KAND,MAMO,IAAI,aAAa,UAAjB,EAA6B;AAClC,UAAI,WAAW,IAAI,IAAJ,CAAS,UAAT,CAAf;AACA,UAAI,QAAJ,EACE,gBAAgB,SAAS,QAAT,CAAkB,QAAlB,CAAhB,CADF,KAGE,gBAAgB,QAAhB;;AAEF,WAAK,QAAL,GAAgB,UAAS,GAAT,EAAc;AAC5B,eAAO,cAAc,UAAd,EAA0B,IAAI,IAAJ,CAAS,GAAT,CAA1B,CAAP;AACD,OAFD;AAGD,KAVM,MAUA,IAAI,aAAa,QAAjB,EAA2B;AAChC,UAAI,OAAO,KAAP,KAAiB,WAArB,EACE,KAAK,QAAL,GAAgB,UAAS,GAAT,EAAc;AAC5B,YAAI,SAAS,KAAK,GAAL,CAAS,EAAT,EAAa,KAAb,CAAb;AACA,eAAO,KAAK,KAAL,CAAW,MAAM,MAAjB,IAA2B,MAAlC;AACD,OAHD;AAIH;;AAED,QAAI,cAAJ,CAAmB,IAAnB;;AAEA,aAAS,QAAT,GAAoB;AAClB,UAAI,SAAS,IAAI,IAAJ,CAAS,gBAAT,EAA2B,MAAxC;;AAEA;AACA,UAAI,gBAAJ;AACA,UAAI,WAAW,IAAI,IAAJ,CAAS,WAAT,CAAf;AACA,UAAI,aAAa,MAAjB,EAAyB;AACvB,kBAAU,iBAAS,GAAT,EAAc;AACtB,iBAAO,cAAc,IAAI,IAAJ,CAAS,CAAC,GAAV,CAAd,CAAP;AACD,SAFD;AAGD,OAJD,MAIO,IAAI,aAAa,UAAjB,EAA6B;AAClC,kBAAU,iBAAS,GAAT,EAAc;AACtB;AACA,iBAAO,CAAC,GAAD,GAAO,IAAd;AACD,SAHD;AAID,OALM,MAKA;AACL,kBAAU,iBAAS,GAAT,EAAc;AAAE,iBAAO,CAAC,GAAR;AAAc,SAAxC;AACD;;AAED,UAAI,IAAI,IAAJ,CAAS,gBAAT,EAA2B,OAA3B,CAAmC,IAAnC,KAA4C,QAAhD,EAA0D;AACxD,eAAO,CAAC,QAAQ,OAAO,IAAf,CAAD,EAAuB,QAAQ,OAAO,EAAf,CAAvB,CAAP;AACD,OAFD,MAEO;AACL,eAAO,QAAQ,OAAO,IAAf,CAAP;AACD;AACF;;AAED,QAAI,gBAAgB,IAApB;;AAEA,QAAI,EAAJ,CAAO,6BAAP,EAAsC,UAAS,KAAT,EAAgB;AACpD,UAAI,CAAC,IAAI,IAAJ,CAAS,UAAT,CAAD,IAAyB,CAAC,IAAI,IAAJ,CAAS,WAAT,CAA9B,EAAqD;AAAA,wBAClC,UADkC;AAAA;AAAA,YAC9C,IAD8C;AAAA,YACxC,EADwC;;AAEnD,YAAI,OAAO,EAAX;AACA,aAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,KAAK,MAAL,CAAY,MAAhC,EAAwC,GAAxC,EAA6C;AAC3C,cAAI,MAAM,KAAK,MAAL,CAAY,CAAZ,CAAV;AACA,cAAI,OAAO,IAAP,IAAe,OAAO,EAA1B,EAA8B;AAC5B,iBAAK,IAAL,CAAU,KAAK,IAAL,CAAU,CAAV,CAAV;AACD;AACF;AACD,aAAK,IAAL;AACA,iBAAS,GAAT,CAAa,IAAb;AACA,wBAAgB,IAAhB;AACD;AACF,KAdD;;AAiBA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,WAAO;AACL,eAAS,mBAAW;AAClB,iBAAS,KAAT;AACD,OAHI;AAIL,cAAQ,kBAAW;AACjB,YAAI,aAAJ,EACE,SAAS,GAAT,CAAa,aAAb;AACH;AAPI,KAAP;AASD;AApHY,CAAf;;AAwHA;AACA,SAAS,QAAT,CAAkB,CAAlB,EAAqB,MAArB,EAA6B;AAC3B,MAAI,MAAM,EAAE,QAAF,EAAV;AACA,SAAO,IAAI,MAAJ,GAAa,MAApB;AACE,UAAM,MAAM,GAAZ;AADF,GAEA,OAAO,GAAP;AACD;;AAED;AACA;AACA,SAAS,aAAT,CAAuB,IAAvB,EAA6B;AAC3B,MAAI,gBAAgB,IAApB,EAA0B;AACxB,WAAO,KAAK,cAAL,KAAwB,GAAxB,GACA,SAAS,KAAK,WAAL,KAAmB,CAA5B,EAA+B,CAA/B,CADA,GACoC,GADpC,GAEA,SAAS,KAAK,UAAL,EAAT,EAA4B,CAA5B,CAFP;AAID,GALD,MAKO;AACL,WAAO,IAAP;AACD;AACF;;;;;;;;;;;;;;ACjJD;;;;AACA;;;;AACA;;IAAY,I;;;;;;;;AAEZ;;;;;;;;;;;;;;;;IAgBa,e,WAAA,e;AAEX,6BAA4C;AAAA,QAAhC,KAAgC,uEAAxB,IAAwB;AAAA,QAAlB,SAAkB,uEAAN,IAAM;;AAAA;;AAC1C,SAAK,WAAL,GAAmB,sBAAnB;AACA,SAAK,QAAL,GAAgB,IAAI,KAAK,mBAAT,CAA6B,KAAK,WAAlC,CAAhB;;AAEA;AACA,SAAK,MAAL,GAAc,IAAd;AACA;AACA,SAAK,IAAL,GAAY,IAAZ;AACA;AACA,SAAK,eAAL,GAAuB,IAAvB;;AAEA,SAAK,UAAL,GAAkB,KAAK,MAAL,CAAY,EAAE,QAAQ,IAAV,EAAZ,EAA8B,SAA9B,CAAlB;;AAEA,SAAK,QAAL,CAAc,KAAd;AACD;;AAED;;;;;;;;;;;;;;;;;6BAaS,K,EAAO;AAAA;;AACd;AACA,UAAI,KAAK,MAAL,KAAgB,KAApB,EACE;AACF;AACA,UAAI,CAAC,KAAK,MAAN,IAAgB,CAAC,KAArB,EACE;;AAEF,UAAI,KAAK,IAAT,EAAe;AACb,aAAK,IAAL,CAAU,GAAV,CAAc,QAAd,EAAwB,KAAK,eAA7B;AACA,aAAK,IAAL,GAAY,IAAZ;AACA,aAAK,eAAL,GAAuB,IAAvB;AACD;;AAED,WAAK,MAAL,GAAc,KAAd;;AAEA,UAAI,KAAJ,EAAW;AACT,aAAK,IAAL,GAAY,qBAAI,KAAJ,EAAW,GAAX,CAAe,WAAf,CAAZ;AACA,YAAI,MAAM,KAAK,IAAL,CAAU,EAAV,CAAa,QAAb,EAAuB,UAAC,CAAD,EAAO;AACtC,gBAAK,WAAL,CAAiB,OAAjB,CAAyB,QAAzB,EAAmC,CAAnC;AACD,SAFS,CAAV;AAGA,aAAK,eAAL,GAAuB,GAAvB;AACD;AACF;;AAED;;;;;;;;;;;;;;;AAcA;;;;;oCAKgB,S,EAAW;AACzB;AACA,aAAO,KAAK,MAAL,CAAY,EAAZ,EACL,KAAK,UAAL,GAAkB,KAAK,UAAvB,GAAoC,IAD/B,EAEL,YAAY,SAAZ,GAAwB,IAFnB,CAAP;AAGD;;AAED;;;;;;;;;;;;;;;wBAYI,Y,EAAc,S,EAAW;AAC3B,UAAI,KAAK,IAAT,EACE,KAAK,IAAL,CAAU,GAAV,CAAc,YAAd,EAA4B,KAAK,eAAL,CAAqB,SAArB,CAA5B;AACH;;AAED;;;;;;;;;;;;;0BAUM,S,EAAW;AACf,UAAI,KAAK,IAAT,EACE,KAAK,GAAL,CAAS,KAAK,CAAd,EAAiB,KAAK,eAAL,CAAqB,SAArB,CAAjB;AACH;;AAED;;;;;;;;;;;;;uBAUG,S,EAAW,Q,EAAU;AACtB,aAAO,KAAK,QAAL,CAAc,EAAd,CAAiB,SAAjB,EAA4B,QAA5B,CAAP;AACD;;AAED;;;;;;;;;;;wBAQI,S,EAAW,Q,EAAU;AACvB,aAAO,KAAK,QAAL,CAAc,GAAd,CAAkB,SAAlB,EAA6B,QAA7B,CAAP;AACD;;AAED;;;;;;;;4BAKQ;AACN,WAAK,QAAL,CAAc,kBAAd;AACA,WAAK,QAAL,CAAc,IAAd;AACD;;;wBAlFW;AACV,aAAO,KAAK,IAAL,GAAY,KAAK,IAAL,CAAU,GAAV,EAAZ,GAA8B,IAArC;AACD;;;;;;AAmFH;;;;;;;;;AASA;;;;;;;;;;;;;;;;;;;;;QCpLgB,M,GAAA,M;QAeA,W,GAAA,W;QAQA,e,GAAA,e;QAoCA,a,GAAA,a;;;;AA3DT,SAAS,MAAT,CAAgB,MAAhB,EAAoC;AAAA,oCAAT,OAAS;AAAT,WAAS;AAAA;;AACzC,OAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,QAAQ,MAA5B,EAAoC,GAApC,EAAyC;AACvC,QAAI,MAAM,QAAQ,CAAR,CAAV;AACA,QAAI,OAAO,GAAP,KAAgB,WAAhB,IAA+B,QAAQ,IAA3C,EACE;;AAEF,SAAK,IAAI,GAAT,IAAgB,GAAhB,EAAqB;AACnB,UAAI,IAAI,cAAJ,CAAmB,GAAnB,CAAJ,EAA6B;AAC3B,eAAO,GAAP,IAAc,IAAI,GAAJ,CAAd;AACD;AACF;AACF;AACD,SAAO,MAAP;AACD;;AAEM,SAAS,WAAT,CAAqB,IAArB,EAA2B;AAChC,OAAK,IAAI,IAAI,CAAb,EAAgB,IAAI,KAAK,MAAzB,EAAiC,GAAjC,EAAsC;AACpC,QAAI,KAAK,CAAL,KAAW,KAAK,IAAE,CAAP,CAAf,EAA0B;AACxB,YAAM,IAAI,KAAJ,CAAU,0CAAV,CAAN;AACD;AACF;AACF;;AAEM,SAAS,eAAT,CAAyB,CAAzB,EAA4B,CAA5B,EAA+B;AACpC,MAAI,MAAM,CAAV;AACA,MAAI,MAAM,CAAV;;AAEA,MAAI,CAAC,CAAL,EAAQ,IAAI,EAAJ;AACR,MAAI,CAAC,CAAL,EAAQ,IAAI,EAAJ;;AAER,MAAI,SAAS,EAAb;AACA,MAAI,SAAS,EAAb;;AAEA,cAAY,CAAZ;AACA,cAAY,CAAZ;;AAEA,SAAO,MAAM,EAAE,MAAR,IAAkB,MAAM,EAAE,MAAjC,EAAyC;AACvC,QAAI,EAAE,GAAF,MAAW,EAAE,GAAF,CAAf,EAAuB;AACrB;AACA;AACD,KAHD,MAGO,IAAI,EAAE,GAAF,IAAS,EAAE,GAAF,CAAb,EAAqB;AAC1B,aAAO,IAAP,CAAY,EAAE,KAAF,CAAZ;AACD,KAFM,MAEA;AACL,aAAO,IAAP,CAAY,EAAE,KAAF,CAAZ;AACD;AACF;;AAED,MAAI,MAAM,EAAE,MAAZ,EACE,SAAS,OAAO,MAAP,CAAc,EAAE,KAAF,CAAQ,GAAR,CAAd,CAAT;AACF,MAAI,MAAM,EAAE,MAAZ,EACE,SAAS,OAAO,MAAP,CAAc,EAAE,KAAF,CAAQ,GAAR,CAAd,CAAT;AACF,SAAO;AACL,aAAS,MADJ;AAEL,WAAO;AAFF,GAAP;AAID;;AAED;AACA;AACO,SAAS,aAAT,CAAuB,EAAvB,EAA2B;AAChC,MAAI,QAAQ,EAAZ;AACA,MAAI,eAAJ;AACA,OAAK,IAAI,IAAT,IAAiB,EAAjB,EAAqB;AACnB,QAAI,GAAG,cAAH,CAAkB,IAAlB,CAAJ,EACE,MAAM,IAAN,CAAW,IAAX;AACF,QAAI,QAAO,GAAG,IAAH,CAAP,MAAqB,QAArB,IAAiC,OAAO,GAAG,IAAH,EAAS,MAAhB,KAA4B,WAAjE,EAA8E;AAC5E,YAAM,IAAI,KAAJ,CAAU,2BAAV,CAAN;AACD,KAFD,MAEO,IAAI,OAAO,MAAP,KAAmB,WAAnB,IAAkC,WAAW,GAAG,IAAH,EAAS,MAA1D,EAAkE;AACvE,YAAM,IAAI,KAAJ,CAAU,8CAAV,CAAN;AACD;AACD,aAAS,GAAG,IAAH,EAAS,MAAlB;AACD;AACD,MAAI,UAAU,EAAd;AACA,MAAI,aAAJ;AACA,OAAK,IAAI,MAAM,CAAf,EAAkB,MAAM,MAAxB,EAAgC,KAAhC,EAAuC;AACrC,WAAO,EAAP;AACA,SAAK,IAAI,MAAM,CAAf,EAAkB,MAAM,MAAM,MAA9B,EAAsC,KAAtC,EAA6C;AAC3C,WAAK,MAAM,GAAN,CAAL,IAAmB,GAAG,MAAM,GAAN,CAAH,EAAe,GAAf,CAAnB;AACD;AACD,YAAQ,IAAR,CAAa,IAAb;AACD;AACD,SAAO,OAAP;AACD;;AAED;;;;;;;IAMa,mB,WAAA,mB;AACX,+BAAY,OAAZ,EAAqB;AAAA;;AACnB,SAAK,QAAL,GAAgB,OAAhB;AACA,SAAK,KAAL,GAAa,EAAb;AACD;;;;uBAEE,S,EAAW,Q,EAAU;AACtB,UAAI,MAAM,KAAK,QAAL,CAAc,EAAd,CAAiB,SAAjB,EAA4B,QAA5B,CAAV;AACA,WAAK,KAAL,CAAW,GAAX,IAAkB,SAAlB;AACA,aAAO,GAAP;AACD;;;wBAEG,S,EAAW,Q,EAAU;AACvB,UAAI,MAAM,KAAK,QAAL,CAAc,GAAd,CAAkB,SAAlB,EAA6B,QAA7B,CAAV;AACA,UAAI,GAAJ,EAAS;AACP,eAAO,KAAK,KAAL,CAAW,GAAX,CAAP;AACD;AACD,aAAO,GAAP;AACD;;;yCAEoB;AAAA;;AACnB,UAAI,eAAe,KAAK,KAAxB;AACA,WAAK,KAAL,GAAa,EAAb;AACA,aAAO,IAAP,CAAY,YAAZ,EAA0B,OAA1B,CAAkC,UAAC,GAAD,EAAS;AACzC,cAAK,QAAL,CAAc,GAAd,CAAkB,aAAa,GAAb,CAAlB,EAAqC,GAArC;AACD,OAFD;AAGD;;;;;;;;;;;;;;;;;;ACpHH;;;;;;;;IAEqB,G;AACnB,eAAY,KAAZ,EAAmB,IAAnB,EAAyB,YAAa,KAAtC,EAA6C;AAAA;;AAC3C,SAAK,MAAL,GAAc,KAAd;AACA,SAAK,KAAL,GAAa,IAAb;AACA,SAAK,MAAL,GAAc,KAAd;AACA,SAAK,OAAL,GAAe,sBAAf;AACD;;;;0BAEK;AACJ,aAAO,KAAK,MAAZ;AACD;;;wBAEG,K,EAAO,YAAa,K,EAAO;AAC7B,UAAI,KAAK,MAAL,KAAgB,KAApB,EAA2B;AACzB;AACA;AACD;AACD,UAAI,WAAW,KAAK,MAApB;AACA,WAAK,MAAL,GAAc,KAAd;AACA;AACA,UAAI,MAAM,EAAV;AACA,UAAI,SAAS,QAAO,KAAP,yCAAO,KAAP,OAAkB,QAA/B,EAAyC;AACvC,aAAK,IAAI,CAAT,IAAc,KAAd,EAAqB;AACnB,cAAI,MAAM,cAAN,CAAqB,CAArB,CAAJ,EACE,IAAI,CAAJ,IAAS,MAAM,CAAN,CAAT;AACH;AACF;AACD,UAAI,QAAJ,GAAe,QAAf;AACA,UAAI,KAAJ,GAAY,KAAZ;AACA,WAAK,OAAL,CAAa,OAAb,CAAqB,QAArB,EAA+B,GAA/B,EAAoC,IAApC;;AAEA;AACA;AACA,UAAI,OAAO,KAAP,IAAgB,OAAO,KAAP,CAAa,aAAjC,EAAgD;AAC9C,eAAO,KAAP,CAAa,aAAb,CACE,mBACG,KAAK,MAAL,CAAY,IAAZ,KAAqB,IAArB,GAA4B,KAAK,MAAL,CAAY,IAAZ,GAAmB,GAA/C,GAAqD,EADxD,IAEE,KAAK,KAHT,EAIE,OAAO,KAAP,KAAkB,WAAlB,GAAgC,IAAhC,GAAuC,KAJzC;AAMD;AACF;;;uBAEE,S,EAAW,Q,EAAU;AACtB,aAAO,KAAK,OAAL,CAAa,EAAb,CAAgB,SAAhB,EAA2B,QAA3B,CAAP;AACD;;;wBAEG,S,EAAW,Q,EAAU;AACvB,aAAO,KAAK,OAAL,CAAa,GAAb,CAAiB,SAAjB,EAA4B,QAA5B,CAAP;AACD;;;;;;kBAjDkB,G", + "file": "generated.js", + "sourceRoot": "", + "sourcesContent": [ + "(function(){function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require==\"function\"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error(\"Cannot find module '\"+o+\"'\");throw f.code=\"MODULE_NOT_FOUND\",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require==\"function\"&&require;for(var o=0;o {\n this._eventRelay.trigger(\"change\", e, this);\n });\n this._varOnChangeSub = sub;\n }\n }\n\n /**\n * Combine the given `extraInfo` (if any) with the handle's default\n * `_extraInfo` (if any).\n * @private\n */\n _mergeExtraInfo(extraInfo) {\n return util.extend({},\n this._extraInfo ? this._extraInfo : null,\n extraInfo ? extraInfo : null);\n }\n\n /**\n * Close the handle. This clears this handle's contribution to the filter set,\n * and unsubscribes all event listeners.\n */\n close() {\n this._emitter.removeAllListeners();\n this.clear();\n this.setGroup(null);\n }\n\n /**\n * Clear this handle's contribution to the filter set.\n *\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `FilterHandle` constructor).\n * \n * @fires FilterHandle#change\n */\n clear(extraInfo) {\n if (!this._filterSet)\n return;\n this._filterSet.clear(this._id);\n this._onChange(extraInfo);\n }\n\n /**\n * Set this handle's contribution to the filter set. This array should consist\n * of the keys of the rows that _should_ be displayed; any keys that are not\n * present in the array will be considered _filtered out_. Note that multiple\n * `FilterHandle` instances in the group may each contribute an array of keys,\n * and only those keys that appear in _all_ of the arrays make it through the\n * filter.\n *\n * @param {string[]} keys - Empty array, or array of keys. To clear the\n * filter, don't pass an empty array; instead, use the\n * {@link FilterHandle#clear} method.\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `FilterHandle` constructor).\n * \n * @fires FilterHandle#change\n */\n set(keys, extraInfo) {\n if (!this._filterSet)\n return;\n this._filterSet.update(this._id, keys);\n this._onChange(extraInfo);\n }\n\n /**\n * @return {string[]|null} - Either: 1) an array of keys that made it through\n * all of the `FilterHandle` instances, or, 2) `null`, which means no filter\n * is being applied (all data should be displayed).\n */\n get filteredKeys() {\n return this._filterSet ? this._filterSet.value : null;\n }\n\n /**\n * Subscribe to events on this `FilterHandle`.\n *\n * @param {string} eventType - Indicates the type of events to listen to.\n * Currently, only `\"change\"` is supported.\n * @param {FilterHandle~listener} listener - The callback function that\n * will be invoked when the event occurs.\n * @return {string} - A token to pass to {@link FilterHandle#off} to cancel\n * this subscription.\n */\n on(eventType, listener) {\n return this._emitter.on(eventType, listener);\n }\n\n /**\n * Cancel event subscriptions created by {@link FilterHandle#on}.\n *\n * @param {string} eventType - The type of event to unsubscribe.\n * @param {string|FilterHandle~listener} listener - Either the callback\n * function previously passed into {@link FilterHandle#on}, or the\n * string that was returned from {@link FilterHandle#on}.\n */\n off(eventType, listener) {\n return this._emitter.off(eventType, listener);\n }\n\n _onChange(extraInfo) {\n if (!this._filterSet)\n return;\n this._filterVar.set(this._filterSet.value, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * @callback FilterHandle~listener\n * @param {Object} event - An object containing details of the event. For\n * `\"change\"` events, this includes the properties `value` (the new\n * value of the filter set, or `null` if no filter set is active),\n * `oldValue` (the previous value of the filter set), and `sender` (the\n * `FilterHandle` instance that made the change).\n */\n\n}\n\n/**\n * @event FilterHandle#change\n * @type {object}\n * @property {object} value - The new value of the filter set, or `null`\n * if no filter set is active.\n * @property {object} oldValue - The previous value of the filter set.\n * @property {FilterHandle} sender - The `FilterHandle` instance that\n * changed the value.\n */\n", + "import { diffSortedLists } from \"./util\";\n\nfunction naturalComparator(a, b) {\n if (a === b) {\n return 0;\n } else if (a < b) {\n return -1;\n } else if (a > b) {\n return 1;\n }\n}\n\n/**\n * @private\n */\nexport default class FilterSet {\n constructor() {\n this.reset();\n }\n\n reset() {\n // Key: handle ID, Value: array of selected keys, or null\n this._handles = {};\n // Key: key string, Value: count of handles that include it\n this._keys = {};\n this._value = null;\n this._activeHandles = 0;\n }\n\n get value() {\n return this._value;\n }\n\n update(handleId, keys) {\n if (keys !== null) {\n keys = keys.slice(0); // clone before sorting\n keys.sort(naturalComparator);\n }\n\n let {added, removed} = diffSortedLists(this._handles[handleId], keys);\n this._handles[handleId] = keys;\n\n for (let i = 0; i < added.length; i++) {\n this._keys[added[i]] = (this._keys[added[i]] || 0) + 1;\n }\n for (let i = 0; i < removed.length; i++) {\n this._keys[removed[i]]--;\n }\n\n this._updateValue(keys);\n }\n\n /**\n * @param {string[]} keys Sorted array of strings that indicate\n * a superset of possible keys.\n * @private\n */\n _updateValue(keys = this._allKeys) {\n let handleCount = Object.keys(this._handles).length;\n if (handleCount === 0) {\n this._value = null;\n } else {\n this._value = [];\n for (let i = 0; i < keys.length; i++) {\n let count = this._keys[keys[i]];\n if (count === handleCount) {\n this._value.push(keys[i]);\n }\n }\n }\n }\n\n clear(handleId) {\n if (typeof(this._handles[handleId]) === \"undefined\") {\n return;\n }\n\n let keys = this._handles[handleId];\n if (!keys) {\n keys = [];\n }\n\n for (let i = 0; i < keys.length; i++) {\n this._keys[keys[i]]--;\n }\n delete this._handles[handleId];\n\n this._updateValue();\n }\n\n get _allKeys() {\n let allKeys = Object.keys(this._keys);\n allKeys.sort(naturalComparator);\n return allKeys;\n }\n}\n", + "import Var from \"./var\";\n\n// Use a global so that multiple copies of crosstalk.js can be loaded and still\n// have groups behave as singletons across all copies.\nglobal.__crosstalk_groups = global.__crosstalk_groups || {};\nlet groups = global.__crosstalk_groups;\n\nexport default function group(groupName) {\n if (groupName && typeof(groupName) === \"string\") {\n if (!groups.hasOwnProperty(groupName)) {\n groups[groupName] = new Group(groupName);\n }\n return groups[groupName];\n } else if (typeof(groupName) === \"object\" && groupName._vars && groupName.var) {\n // Appears to already be a group object\n return groupName;\n } else if (Array.isArray(groupName) &&\n groupName.length == 1 &&\n typeof(groupName[0]) === \"string\") {\n return group(groupName[0]);\n } else {\n throw new Error(\"Invalid groupName argument\");\n }\n}\n\nclass Group {\n constructor(name) {\n this.name = name;\n this._vars = {};\n }\n\n var(name) {\n if (!name || typeof(name) !== \"string\") {\n throw new Error(\"Invalid var name\");\n }\n\n if (!this._vars.hasOwnProperty(name))\n this._vars[name] = new Var(this, name);\n return this._vars[name];\n }\n\n has(name) {\n if (!name || typeof(name) !== \"string\") {\n throw new Error(\"Invalid var name\");\n }\n\n return this._vars.hasOwnProperty(name);\n }\n}\n", + "import group from \"./group\";\nimport { SelectionHandle } from \"./selection\";\nimport { FilterHandle } from \"./filter\";\nimport { bind } from \"./input\";\nimport \"./input_selectize\";\nimport \"./input_checkboxgroup\";\nimport \"./input_slider\";\n\nconst defaultGroup = group(\"default\");\n\nfunction var_(name) {\n return defaultGroup.var(name);\n}\n\nfunction has(name) {\n return defaultGroup.has(name);\n}\n\nif (global.Shiny) {\n global.Shiny.addCustomMessageHandler(\"update-client-value\", function(message) {\n if (typeof(message.group) === \"string\") {\n group(message.group).var(message.name).set(message.value);\n } else {\n var_(message.name).set(message.value);\n }\n });\n}\n\nconst crosstalk = {\n group: group,\n var: var_,\n has: has,\n SelectionHandle: SelectionHandle,\n FilterHandle: FilterHandle,\n bind: bind\n};\n\n/**\n * @namespace crosstalk\n */\nexport default crosstalk;\nglobal.crosstalk = crosstalk;\n", + "let $ = global.jQuery;\n\nlet bindings = {};\n\nexport function register(reg) {\n bindings[reg.className] = reg;\n if (global.document && global.document.readyState !== \"complete\") {\n $(() => {\n bind();\n });\n } else if (global.document) {\n setTimeout(bind, 100);\n }\n}\n\nexport function bind() {\n Object.keys(bindings).forEach(function(className) {\n let binding = bindings[className];\n $(\".\" + binding.className).not(\".crosstalk-input-bound\").each(function(i, el) {\n bindInstance(binding, el);\n });\n });\n}\n\n// Escape jQuery identifier\nfunction $escape(val) {\n return val.replace(/([!\"#$%&'()*+,./:;<=>?@[\\\\\\]^`{|}~])/g, \"\\\\$1\");\n}\n\nfunction bindEl(el) {\n let $el = $(el);\n Object.keys(bindings).forEach(function(className) {\n if ($el.hasClass(className) && !$el.hasClass(\"crosstalk-input-bound\")) {\n let binding = bindings[className];\n bindInstance(binding, el);\n }\n });\n}\n\nfunction bindInstance(binding, el) {\n let jsonEl = $(el).find(\"script[type='application/json'][data-for='\" + $escape(el.id) + \"']\");\n let data = JSON.parse(jsonEl[0].innerText);\n\n let instance = binding.factory(el, data);\n $(el).data(\"crosstalk-instance\", instance);\n $(el).addClass(\"crosstalk-input-bound\");\n}\n\nif (global.Shiny) {\n let inputBinding = new global.Shiny.InputBinding();\n let $ = global.jQuery;\n $.extend(inputBinding, {\n find: function(scope) {\n return $(scope).find(\".crosstalk-input\");\n },\n initialize: function(el) {\n if (!$(el).hasClass(\"crosstalk-input-bound\")) {\n bindEl(el);\n }\n },\n getId: function(el) {\n return el.id;\n },\n getValue: function(el) {\n\n },\n setValue: function(el, value) {\n\n },\n receiveMessage: function(el, data) {\n\n },\n subscribe: function(el, callback) {\n $(el).data(\"crosstalk-instance\").resume();\n },\n unsubscribe: function(el) {\n $(el).data(\"crosstalk-instance\").suspend();\n }\n });\n global.Shiny.inputBindings.register(inputBinding, \"crosstalk.inputBinding\");\n}\n", + "import * as input from \"./input\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\n\ninput.register({\n className: \"crosstalk-input-checkboxgroup\",\n\n factory: function(el, data) {\n /*\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n let ctHandle = new FilterHandle(data.group);\n\n let lastKnownKeys;\n let $el = $(el);\n $el.on(\"change\", \"input[type='checkbox']\", function() {\n let checked = $el.find(\"input[type='checkbox']:checked\");\n if (checked.length === 0) {\n lastKnownKeys = null;\n ctHandle.clear();\n } else {\n let keys = {};\n checked.each(function() {\n data.map[this.value].forEach(function(key) {\n keys[key] = true;\n });\n });\n let keyArray = Object.keys(keys);\n keyArray.sort();\n lastKnownKeys = keyArray;\n ctHandle.set(keyArray);\n }\n });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n", + "import * as input from \"./input\";\nimport * as util from \"./util\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\n\ninput.register({\n className: \"crosstalk-input-select\",\n\n factory: function(el, data) {\n /*\n * items: {value: [...], label: [...]}\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n\n let first = [{value: \"\", label: \"(All)\"}];\n let items = util.dataframeToD3(data.items);\n let opts = {\n options: first.concat(items),\n valueField: \"value\",\n labelField: \"label\",\n searchField: \"label\"\n };\n\n let select = $(el).find(\"select\")[0];\n\n let selectize = $(select).selectize(opts)[0].selectize;\n\n let ctHandle = new FilterHandle(data.group);\n\n let lastKnownKeys;\n selectize.on(\"change\", function() {\n if (selectize.items.length === 0) {\n lastKnownKeys = null;\n ctHandle.clear();\n } else {\n let keys = {};\n selectize.items.forEach(function(group) {\n data.map[group].forEach(function(key) {\n keys[key] = true;\n });\n });\n let keyArray = Object.keys(keys);\n keyArray.sort();\n lastKnownKeys = keyArray;\n ctHandle.set(keyArray);\n }\n });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n", + "import * as input from \"./input\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\nlet strftime = global.strftime;\n\ninput.register({\n className: \"crosstalk-input-slider\",\n\n factory: function(el, data) {\n /*\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n let ctHandle = new FilterHandle(data.group);\n\n let opts = {};\n let $el = $(el).find(\"input\");\n let dataType = $el.data(\"data-type\");\n let timeFormat = $el.data(\"time-format\");\n let round = $el.data(\"round\");\n let timeFormatter;\n\n // Set up formatting functions\n if (dataType === \"date\") {\n timeFormatter = strftime.utc();\n opts.prettify = function(num) {\n return timeFormatter(timeFormat, new Date(num));\n };\n\n } else if (dataType === \"datetime\") {\n let timezone = $el.data(\"timezone\");\n if (timezone)\n timeFormatter = strftime.timezone(timezone);\n else\n timeFormatter = strftime;\n\n opts.prettify = function(num) {\n return timeFormatter(timeFormat, new Date(num));\n };\n } else if (dataType === \"number\") {\n if (typeof round !== \"undefined\")\n opts.prettify = function(num) {\n let factor = Math.pow(10, round);\n return Math.round(num * factor) / factor;\n };\n }\n\n $el.ionRangeSlider(opts);\n\n function getValue() {\n let result = $el.data(\"ionRangeSlider\").result;\n\n // Function for converting numeric value from slider to appropriate type.\n let convert;\n let dataType = $el.data(\"data-type\");\n if (dataType === \"date\") {\n convert = function(val) {\n return formatDateUTC(new Date(+val));\n };\n } else if (dataType === \"datetime\") {\n convert = function(val) {\n // Convert ms to s\n return +val / 1000;\n };\n } else {\n convert = function(val) { return +val; };\n }\n\n if ($el.data(\"ionRangeSlider\").options.type === \"double\") {\n return [convert(result.from), convert(result.to)];\n } else {\n return convert(result.from);\n }\n }\n\n let lastKnownKeys = null;\n\n $el.on(\"change.crosstalkSliderInput\", function(event) {\n if (!$el.data(\"updating\") && !$el.data(\"animating\")) {\n let [from, to] = getValue();\n let keys = [];\n for (let i = 0; i < data.values.length; i++) {\n let val = data.values[i];\n if (val >= from && val <= to) {\n keys.push(data.keys[i]);\n }\n }\n keys.sort();\n ctHandle.set(keys);\n lastKnownKeys = keys;\n }\n });\n\n\n // let $el = $(el);\n // $el.on(\"change\", \"input[type=\"checkbox\"]\", function() {\n // let checked = $el.find(\"input[type=\"checkbox\"]:checked\");\n // if (checked.length === 0) {\n // ctHandle.clear();\n // } else {\n // let keys = {};\n // checked.each(function() {\n // data.map[this.value].forEach(function(key) {\n // keys[key] = true;\n // });\n // });\n // let keyArray = Object.keys(keys);\n // keyArray.sort();\n // ctHandle.set(keyArray);\n // }\n // });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n\n\n// Convert a number to a string with leading zeros\nfunction padZeros(n, digits) {\n let str = n.toString();\n while (str.length < digits)\n str = \"0\" + str;\n return str;\n}\n\n// Given a Date object, return a string in yyyy-mm-dd format, using the\n// UTC date. This may be a day off from the date in the local time zone.\nfunction formatDateUTC(date) {\n if (date instanceof Date) {\n return date.getUTCFullYear() + \"-\" +\n padZeros(date.getUTCMonth()+1, 2) + \"-\" +\n padZeros(date.getUTCDate(), 2);\n\n } else {\n return null;\n }\n}\n", + "import Events from \"./events\";\nimport grp from \"./group\";\nimport * as util from \"./util\";\n\n/**\n * Use this class to read and write (and listen for changes to) the selection\n * for a Crosstalk group. This is intended to be used for linked brushing.\n *\n * If two (or more) `SelectionHandle` instances in the same webpage share the\n * same group name, they will share the same state. Setting the selection using\n * one `SelectionHandle` instance will result in the `value` property instantly\n * changing across the others, and `\"change\"` event listeners on all instances\n * (including the one that initiated the sending) will fire.\n *\n * @param {string} [group] - The name of the Crosstalk group, or if none,\n * null or undefined (or any other falsy value). This can be changed later\n * via the [SelectionHandle#setGroup](#setGroup) method.\n * @param {Object} [extraInfo] - An object whose properties will be copied to\n * the event object whenever an event is emitted.\n */\nexport class SelectionHandle {\n\n constructor(group = null, extraInfo = null) {\n this._eventRelay = new Events();\n this._emitter = new util.SubscriptionTracker(this._eventRelay);\n\n // Name of the group we're currently tracking, if any. Can change over time.\n this._group = null;\n // The Var we're currently tracking, if any. Can change over time.\n this._var = null;\n // The event handler subscription we currently have on var.on(\"change\").\n this._varOnChangeSub = null;\n\n this._extraInfo = util.extend({ sender: this }, extraInfo);\n\n this.setGroup(group);\n }\n\n /**\n * Changes the Crosstalk group membership of this SelectionHandle. The group\n * being switched away from (if any) will not have its selection value\n * modified as a result of calling `setGroup`, even if this handle was the\n * most recent handle to set the selection of the group.\n *\n * The group being switched to (if any) will also not have its selection value\n * modified as a result of calling `setGroup`. If you want to set the\n * selection value of the new group, call `set` explicitly.\n *\n * @param {string} group - The name of the Crosstalk group, or null (or\n * undefined) to clear the group.\n */\n setGroup(group) {\n // If group is unchanged, do nothing\n if (this._group === group)\n return;\n // Treat null, undefined, and other falsy values the same\n if (!this._group && !group)\n return;\n\n if (this._var) {\n this._var.off(\"change\", this._varOnChangeSub);\n this._var = null;\n this._varOnChangeSub = null;\n }\n\n this._group = group;\n\n if (group) {\n this._var = grp(group).var(\"selection\");\n let sub = this._var.on(\"change\", (e) => {\n this._eventRelay.trigger(\"change\", e, this);\n });\n this._varOnChangeSub = sub;\n }\n }\n\n /**\n * Retrieves the current selection for the group represented by this\n * `SelectionHandle`.\n *\n * - If no selection is active, then this value will be falsy.\n * - If a selection is active, but no data points are selected, then this\n * value will be an empty array.\n * - If a selection is active, and data points are selected, then the keys\n * of the selected data points will be present in the array.\n */\n get value() {\n return this._var ? this._var.get() : null;\n }\n\n /**\n * Combines the given `extraInfo` (if any) with the handle's default\n * `_extraInfo` (if any).\n * @private\n */\n _mergeExtraInfo(extraInfo) {\n // Important incidental effect: shallow clone is returned\n return util.extend({},\n this._extraInfo ? this._extraInfo : null,\n extraInfo ? extraInfo : null);\n }\n\n /**\n * Overwrites the current selection for the group, and raises the `\"change\"`\n * event among all of the group's '`SelectionHandle` instances (including\n * this one).\n *\n * @fires SelectionHandle#change\n * @param {string[]} selectedKeys - Falsy, empty array, or array of keys (see\n * {@link SelectionHandle#value}).\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `SelectionHandle` constructor).\n */\n set(selectedKeys, extraInfo) {\n if (this._var)\n this._var.set(selectedKeys, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * Overwrites the current selection for the group, and raises the `\"change\"`\n * event among all of the group's '`SelectionHandle` instances (including\n * this one).\n *\n * @fires SelectionHandle#change\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any that were passed\n * into the `SelectionHandle` constructor).\n */\n clear(extraInfo) {\n if (this._var)\n this.set(void 0, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * Subscribes to events on this `SelectionHandle`.\n *\n * @param {string} eventType - Indicates the type of events to listen to.\n * Currently, only `\"change\"` is supported.\n * @param {SelectionHandle~listener} listener - The callback function that\n * will be invoked when the event occurs.\n * @return {string} - A token to pass to {@link SelectionHandle#off} to cancel\n * this subscription.\n */\n on(eventType, listener) {\n return this._emitter.on(eventType, listener);\n }\n\n /**\n * Cancels event subscriptions created by {@link SelectionHandle#on}.\n *\n * @param {string} eventType - The type of event to unsubscribe.\n * @param {string|SelectionHandle~listener} listener - Either the callback\n * function previously passed into {@link SelectionHandle#on}, or the\n * string that was returned from {@link SelectionHandle#on}.\n */\n off(eventType, listener) {\n return this._emitter.off(eventType, listener);\n }\n\n /**\n * Shuts down the `SelectionHandle` object.\n *\n * Removes all event listeners that were added through this handle.\n */\n close() {\n this._emitter.removeAllListeners();\n this.setGroup(null);\n }\n}\n\n/**\n * @callback SelectionHandle~listener\n * @param {Object} event - An object containing details of the event. For\n * `\"change\"` events, this includes the properties `value` (the new\n * value of the selection, or `undefined` if no selection is active),\n * `oldValue` (the previous value of the selection), and `sender` (the\n * `SelectionHandle` instance that made the change).\n */\n\n/**\n * @event SelectionHandle#change\n * @type {object}\n * @property {object} value - The new value of the selection, or `undefined`\n * if no selection is active.\n * @property {object} oldValue - The previous value of the selection.\n * @property {SelectionHandle} sender - The `SelectionHandle` instance that\n * changed the value.\n */\n", + "export function extend(target, ...sources) {\n for (let i = 0; i < sources.length; i++) {\n let src = sources[i];\n if (typeof(src) === \"undefined\" || src === null)\n continue;\n\n for (let key in src) {\n if (src.hasOwnProperty(key)) {\n target[key] = src[key];\n }\n }\n }\n return target;\n}\n\nexport function checkSorted(list) {\n for (let i = 1; i < list.length; i++) {\n if (list[i] <= list[i-1]) {\n throw new Error(\"List is not sorted or contains duplicate\");\n }\n }\n}\n\nexport function diffSortedLists(a, b) {\n let i_a = 0;\n let i_b = 0;\n\n if (!a) a = [];\n if (!b) b = [];\n\n let a_only = [];\n let b_only = [];\n\n checkSorted(a);\n checkSorted(b);\n\n while (i_a < a.length && i_b < b.length) {\n if (a[i_a] === b[i_b]) {\n i_a++;\n i_b++;\n } else if (a[i_a] < b[i_b]) {\n a_only.push(a[i_a++]);\n } else {\n b_only.push(b[i_b++]);\n }\n }\n\n if (i_a < a.length)\n a_only = a_only.concat(a.slice(i_a));\n if (i_b < b.length)\n b_only = b_only.concat(b.slice(i_b));\n return {\n removed: a_only,\n added: b_only\n };\n}\n\n// Convert from wide: { colA: [1,2,3], colB: [4,5,6], ... }\n// to long: [ {colA: 1, colB: 4}, {colA: 2, colB: 5}, ... ]\nexport function dataframeToD3(df) {\n let names = [];\n let length;\n for (let name in df) {\n if (df.hasOwnProperty(name))\n names.push(name);\n if (typeof(df[name]) !== \"object\" || typeof(df[name].length) === \"undefined\") {\n throw new Error(\"All fields must be arrays\");\n } else if (typeof(length) !== \"undefined\" && length !== df[name].length) {\n throw new Error(\"All fields must be arrays of the same length\");\n }\n length = df[name].length;\n }\n let results = [];\n let item;\n for (let row = 0; row < length; row++) {\n item = {};\n for (let col = 0; col < names.length; col++) {\n item[names[col]] = df[names[col]][row];\n }\n results.push(item);\n }\n return results;\n}\n\n/**\n * Keeps track of all event listener additions/removals and lets all active\n * listeners be removed with a single operation.\n *\n * @private\n */\nexport class SubscriptionTracker {\n constructor(emitter) {\n this._emitter = emitter;\n this._subs = {};\n }\n\n on(eventType, listener) {\n let sub = this._emitter.on(eventType, listener);\n this._subs[sub] = eventType;\n return sub;\n }\n\n off(eventType, listener) {\n let sub = this._emitter.off(eventType, listener);\n if (sub) {\n delete this._subs[sub];\n }\n return sub;\n }\n\n removeAllListeners() {\n let current_subs = this._subs;\n this._subs = {};\n Object.keys(current_subs).forEach((sub) => {\n this._emitter.off(current_subs[sub], sub);\n });\n }\n}\n", + "import Events from \"./events\";\n\nexport default class Var {\n constructor(group, name, /*optional*/ value) {\n this._group = group;\n this._name = name;\n this._value = value;\n this._events = new Events();\n }\n\n get() {\n return this._value;\n }\n\n set(value, /*optional*/ event) {\n if (this._value === value) {\n // Do nothing; the value hasn't changed\n return;\n }\n let oldValue = this._value;\n this._value = value;\n // Alert JavaScript listeners that the value has changed\n let evt = {};\n if (event && typeof(event) === \"object\") {\n for (let k in event) {\n if (event.hasOwnProperty(k))\n evt[k] = event[k];\n }\n }\n evt.oldValue = oldValue;\n evt.value = value;\n this._events.trigger(\"change\", evt, this);\n\n // TODO: Make this extensible, to let arbitrary back-ends know that\n // something has changed\n if (global.Shiny && global.Shiny.onInputChange) {\n global.Shiny.onInputChange(\n \".clientValue-\" +\n (this._group.name !== null ? this._group.name + \"-\" : \"\") +\n this._name,\n typeof(value) === \"undefined\" ? null : value\n );\n }\n }\n\n on(eventType, listener) {\n return this._events.on(eventType, listener);\n }\n\n off(eventType, listener) {\n return this._events.off(eventType, listener);\n }\n}\n" + ] +} \ No newline at end of file diff --git a/docs/site_libs/crosstalk-1.2.1/js/crosstalk.min.js b/docs/site_libs/crosstalk-1.2.1/js/crosstalk.min.js new file mode 100644 index 00000000..b7ec0ac9 --- /dev/null +++ b/docs/site_libs/crosstalk-1.2.1/js/crosstalk.min.js @@ -0,0 +1,2 @@ +!function o(u,a,l){function s(n,e){if(!a[n]){if(!u[n]){var t="function"==typeof require&&require;if(!e&&t)return t(n,!0);if(f)return f(n,!0);var r=new Error("Cannot find module '"+n+"'");throw r.code="MODULE_NOT_FOUND",r}var i=a[n]={exports:{}};u[n][0].call(i.exports,function(e){var t=u[n][1][e];return s(t||e)},i,i.exports,o,u,a,l)}return a[n].exports}for(var f="function"==typeof require&&require,e=0;e?@[\\\]^`{|}~])/g,"\\$1")+"']"),r=JSON.parse(n[0].innerText),i=e.factory(t,r);o(t).data("crosstalk-instance",i),o(t).addClass("crosstalk-input-bound")}if(t.Shiny){var e=new t.Shiny.InputBinding,u=t.jQuery;u.extend(e,{find:function(e){return u(e).find(".crosstalk-input")},initialize:function(e){var t,n;u(e).hasClass("crosstalk-input-bound")||(n=o(t=e),Object.keys(r).forEach(function(e){n.hasClass(e)&&!n.hasClass("crosstalk-input-bound")&&i(r[e],t)}))},getId:function(e){return e.id},getValue:function(e){},setValue:function(e,t){},receiveMessage:function(e,t){},subscribe:function(e,t){u(e).data("crosstalk-instance").resume()},unsubscribe:function(e){u(e).data("crosstalk-instance").suspend()}}),t.Shiny.inputBindings.register(e,"crosstalk.inputBinding")}}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{}],7:[function(r,e,t){(function(e){"use strict";var t=function(e){{if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}}(r("./input")),n=r("./filter");var a=e.jQuery;t.register({className:"crosstalk-input-checkboxgroup",factory:function(e,r){var i=new n.FilterHandle(r.group),o=void 0,u=a(e);return u.on("change","input[type='checkbox']",function(){var e=u.find("input[type='checkbox']:checked");if(0===e.length)o=null,i.clear();else{var t={};e.each(function(){r.map[this.value].forEach(function(e){t[e]=!0})});var n=Object.keys(t);n.sort(),o=n,i.set(n)}}),{suspend:function(){i.clear()},resume:function(){o&&i.set(o)}}}})}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./filter":2,"./input":6}],8:[function(r,e,t){(function(e){"use strict";var t=n(r("./input")),l=n(r("./util")),s=r("./filter");function n(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}var f=e.jQuery;t.register({className:"crosstalk-input-select",factory:function(e,n){var t=l.dataframeToD3(n.items),r={options:[{value:"",label:"(All)"}].concat(t),valueField:"value",labelField:"label",searchField:"label"},i=f(e).find("select")[0],o=f(i).selectize(r)[0].selectize,u=new s.FilterHandle(n.group),a=void 0;return o.on("change",function(){if(0===o.items.length)a=null,u.clear();else{var t={};o.items.forEach(function(e){n.map[e].forEach(function(e){t[e]=!0})});var e=Object.keys(t);e.sort(),a=e,u.set(e)}}),{suspend:function(){u.clear()},resume:function(){a&&u.set(a)}}}})}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{"./filter":2,"./input":6,"./util":11}],9:[function(n,e,t){(function(e){"use strict";var d=function(e,t){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return function(e,t){var n=[],r=!0,i=!1,o=void 0;try{for(var u,a=e[Symbol.iterator]();!(r=(u=a.next()).done)&&(n.push(u.value),!t||n.length!==t);r=!0);}catch(e){i=!0,o=e}finally{try{!r&&a.return&&a.return()}finally{if(i)throw o}}return n}(e,t);throw new TypeError("Invalid attempt to destructure non-iterable instance")},t=function(e){{if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}}(n("./input")),a=n("./filter");var v=e.jQuery,p=e.strftime;function y(e,t){for(var n=e.toString();n.length {\n this._eventRelay.trigger(\"change\", e, this);\n });\n this._varOnChangeSub = sub;\n }\n }\n\n /**\n * Combine the given `extraInfo` (if any) with the handle's default\n * `_extraInfo` (if any).\n * @private\n */\n _mergeExtraInfo(extraInfo) {\n return util.extend({},\n this._extraInfo ? this._extraInfo : null,\n extraInfo ? extraInfo : null);\n }\n\n /**\n * Close the handle. This clears this handle's contribution to the filter set,\n * and unsubscribes all event listeners.\n */\n close() {\n this._emitter.removeAllListeners();\n this.clear();\n this.setGroup(null);\n }\n\n /**\n * Clear this handle's contribution to the filter set.\n *\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `FilterHandle` constructor).\n * \n * @fires FilterHandle#change\n */\n clear(extraInfo) {\n if (!this._filterSet)\n return;\n this._filterSet.clear(this._id);\n this._onChange(extraInfo);\n }\n\n /**\n * Set this handle's contribution to the filter set. This array should consist\n * of the keys of the rows that _should_ be displayed; any keys that are not\n * present in the array will be considered _filtered out_. Note that multiple\n * `FilterHandle` instances in the group may each contribute an array of keys,\n * and only those keys that appear in _all_ of the arrays make it through the\n * filter.\n *\n * @param {string[]} keys - Empty array, or array of keys. To clear the\n * filter, don't pass an empty array; instead, use the\n * {@link FilterHandle#clear} method.\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `FilterHandle` constructor).\n * \n * @fires FilterHandle#change\n */\n set(keys, extraInfo) {\n if (!this._filterSet)\n return;\n this._filterSet.update(this._id, keys);\n this._onChange(extraInfo);\n }\n\n /**\n * @return {string[]|null} - Either: 1) an array of keys that made it through\n * all of the `FilterHandle` instances, or, 2) `null`, which means no filter\n * is being applied (all data should be displayed).\n */\n get filteredKeys() {\n return this._filterSet ? this._filterSet.value : null;\n }\n\n /**\n * Subscribe to events on this `FilterHandle`.\n *\n * @param {string} eventType - Indicates the type of events to listen to.\n * Currently, only `\"change\"` is supported.\n * @param {FilterHandle~listener} listener - The callback function that\n * will be invoked when the event occurs.\n * @return {string} - A token to pass to {@link FilterHandle#off} to cancel\n * this subscription.\n */\n on(eventType, listener) {\n return this._emitter.on(eventType, listener);\n }\n\n /**\n * Cancel event subscriptions created by {@link FilterHandle#on}.\n *\n * @param {string} eventType - The type of event to unsubscribe.\n * @param {string|FilterHandle~listener} listener - Either the callback\n * function previously passed into {@link FilterHandle#on}, or the\n * string that was returned from {@link FilterHandle#on}.\n */\n off(eventType, listener) {\n return this._emitter.off(eventType, listener);\n }\n\n _onChange(extraInfo) {\n if (!this._filterSet)\n return;\n this._filterVar.set(this._filterSet.value, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * @callback FilterHandle~listener\n * @param {Object} event - An object containing details of the event. For\n * `\"change\"` events, this includes the properties `value` (the new\n * value of the filter set, or `null` if no filter set is active),\n * `oldValue` (the previous value of the filter set), and `sender` (the\n * `FilterHandle` instance that made the change).\n */\n\n}\n\n/**\n * @event FilterHandle#change\n * @type {object}\n * @property {object} value - The new value of the filter set, or `null`\n * if no filter set is active.\n * @property {object} oldValue - The previous value of the filter set.\n * @property {FilterHandle} sender - The `FilterHandle` instance that\n * changed the value.\n */\n","import { diffSortedLists } from \"./util\";\n\nfunction naturalComparator(a, b) {\n if (a === b) {\n return 0;\n } else if (a < b) {\n return -1;\n } else if (a > b) {\n return 1;\n }\n}\n\n/**\n * @private\n */\nexport default class FilterSet {\n constructor() {\n this.reset();\n }\n\n reset() {\n // Key: handle ID, Value: array of selected keys, or null\n this._handles = {};\n // Key: key string, Value: count of handles that include it\n this._keys = {};\n this._value = null;\n this._activeHandles = 0;\n }\n\n get value() {\n return this._value;\n }\n\n update(handleId, keys) {\n if (keys !== null) {\n keys = keys.slice(0); // clone before sorting\n keys.sort(naturalComparator);\n }\n\n let {added, removed} = diffSortedLists(this._handles[handleId], keys);\n this._handles[handleId] = keys;\n\n for (let i = 0; i < added.length; i++) {\n this._keys[added[i]] = (this._keys[added[i]] || 0) + 1;\n }\n for (let i = 0; i < removed.length; i++) {\n this._keys[removed[i]]--;\n }\n\n this._updateValue(keys);\n }\n\n /**\n * @param {string[]} keys Sorted array of strings that indicate\n * a superset of possible keys.\n * @private\n */\n _updateValue(keys = this._allKeys) {\n let handleCount = Object.keys(this._handles).length;\n if (handleCount === 0) {\n this._value = null;\n } else {\n this._value = [];\n for (let i = 0; i < keys.length; i++) {\n let count = this._keys[keys[i]];\n if (count === handleCount) {\n this._value.push(keys[i]);\n }\n }\n }\n }\n\n clear(handleId) {\n if (typeof(this._handles[handleId]) === \"undefined\") {\n return;\n }\n\n let keys = this._handles[handleId];\n if (!keys) {\n keys = [];\n }\n\n for (let i = 0; i < keys.length; i++) {\n this._keys[keys[i]]--;\n }\n delete this._handles[handleId];\n\n this._updateValue();\n }\n\n get _allKeys() {\n let allKeys = Object.keys(this._keys);\n allKeys.sort(naturalComparator);\n return allKeys;\n }\n}\n","import Var from \"./var\";\n\n// Use a global so that multiple copies of crosstalk.js can be loaded and still\n// have groups behave as singletons across all copies.\nglobal.__crosstalk_groups = global.__crosstalk_groups || {};\nlet groups = global.__crosstalk_groups;\n\nexport default function group(groupName) {\n if (groupName && typeof(groupName) === \"string\") {\n if (!groups.hasOwnProperty(groupName)) {\n groups[groupName] = new Group(groupName);\n }\n return groups[groupName];\n } else if (typeof(groupName) === \"object\" && groupName._vars && groupName.var) {\n // Appears to already be a group object\n return groupName;\n } else if (Array.isArray(groupName) &&\n groupName.length == 1 &&\n typeof(groupName[0]) === \"string\") {\n return group(groupName[0]);\n } else {\n throw new Error(\"Invalid groupName argument\");\n }\n}\n\nclass Group {\n constructor(name) {\n this.name = name;\n this._vars = {};\n }\n\n var(name) {\n if (!name || typeof(name) !== \"string\") {\n throw new Error(\"Invalid var name\");\n }\n\n if (!this._vars.hasOwnProperty(name))\n this._vars[name] = new Var(this, name);\n return this._vars[name];\n }\n\n has(name) {\n if (!name || typeof(name) !== \"string\") {\n throw new Error(\"Invalid var name\");\n }\n\n return this._vars.hasOwnProperty(name);\n }\n}\n","import group from \"./group\";\nimport { SelectionHandle } from \"./selection\";\nimport { FilterHandle } from \"./filter\";\nimport { bind } from \"./input\";\nimport \"./input_selectize\";\nimport \"./input_checkboxgroup\";\nimport \"./input_slider\";\n\nconst defaultGroup = group(\"default\");\n\nfunction var_(name) {\n return defaultGroup.var(name);\n}\n\nfunction has(name) {\n return defaultGroup.has(name);\n}\n\nif (global.Shiny) {\n global.Shiny.addCustomMessageHandler(\"update-client-value\", function(message) {\n if (typeof(message.group) === \"string\") {\n group(message.group).var(message.name).set(message.value);\n } else {\n var_(message.name).set(message.value);\n }\n });\n}\n\nconst crosstalk = {\n group: group,\n var: var_,\n has: has,\n SelectionHandle: SelectionHandle,\n FilterHandle: FilterHandle,\n bind: bind\n};\n\n/**\n * @namespace crosstalk\n */\nexport default crosstalk;\nglobal.crosstalk = crosstalk;\n","let $ = global.jQuery;\n\nlet bindings = {};\n\nexport function register(reg) {\n bindings[reg.className] = reg;\n if (global.document && global.document.readyState !== \"complete\") {\n $(() => {\n bind();\n });\n } else if (global.document) {\n setTimeout(bind, 100);\n }\n}\n\nexport function bind() {\n Object.keys(bindings).forEach(function(className) {\n let binding = bindings[className];\n $(\".\" + binding.className).not(\".crosstalk-input-bound\").each(function(i, el) {\n bindInstance(binding, el);\n });\n });\n}\n\n// Escape jQuery identifier\nfunction $escape(val) {\n return val.replace(/([!\"#$%&'()*+,./:;<=>?@[\\\\\\]^`{|}~])/g, \"\\\\$1\");\n}\n\nfunction bindEl(el) {\n let $el = $(el);\n Object.keys(bindings).forEach(function(className) {\n if ($el.hasClass(className) && !$el.hasClass(\"crosstalk-input-bound\")) {\n let binding = bindings[className];\n bindInstance(binding, el);\n }\n });\n}\n\nfunction bindInstance(binding, el) {\n let jsonEl = $(el).find(\"script[type='application/json'][data-for='\" + $escape(el.id) + \"']\");\n let data = JSON.parse(jsonEl[0].innerText);\n\n let instance = binding.factory(el, data);\n $(el).data(\"crosstalk-instance\", instance);\n $(el).addClass(\"crosstalk-input-bound\");\n}\n\nif (global.Shiny) {\n let inputBinding = new global.Shiny.InputBinding();\n let $ = global.jQuery;\n $.extend(inputBinding, {\n find: function(scope) {\n return $(scope).find(\".crosstalk-input\");\n },\n initialize: function(el) {\n if (!$(el).hasClass(\"crosstalk-input-bound\")) {\n bindEl(el);\n }\n },\n getId: function(el) {\n return el.id;\n },\n getValue: function(el) {\n\n },\n setValue: function(el, value) {\n\n },\n receiveMessage: function(el, data) {\n\n },\n subscribe: function(el, callback) {\n $(el).data(\"crosstalk-instance\").resume();\n },\n unsubscribe: function(el) {\n $(el).data(\"crosstalk-instance\").suspend();\n }\n });\n global.Shiny.inputBindings.register(inputBinding, \"crosstalk.inputBinding\");\n}\n","import * as input from \"./input\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\n\ninput.register({\n className: \"crosstalk-input-checkboxgroup\",\n\n factory: function(el, data) {\n /*\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n let ctHandle = new FilterHandle(data.group);\n\n let lastKnownKeys;\n let $el = $(el);\n $el.on(\"change\", \"input[type='checkbox']\", function() {\n let checked = $el.find(\"input[type='checkbox']:checked\");\n if (checked.length === 0) {\n lastKnownKeys = null;\n ctHandle.clear();\n } else {\n let keys = {};\n checked.each(function() {\n data.map[this.value].forEach(function(key) {\n keys[key] = true;\n });\n });\n let keyArray = Object.keys(keys);\n keyArray.sort();\n lastKnownKeys = keyArray;\n ctHandle.set(keyArray);\n }\n });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n","import * as input from \"./input\";\nimport * as util from \"./util\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\n\ninput.register({\n className: \"crosstalk-input-select\",\n\n factory: function(el, data) {\n /*\n * items: {value: [...], label: [...]}\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n\n let first = [{value: \"\", label: \"(All)\"}];\n let items = util.dataframeToD3(data.items);\n let opts = {\n options: first.concat(items),\n valueField: \"value\",\n labelField: \"label\",\n searchField: \"label\"\n };\n\n let select = $(el).find(\"select\")[0];\n\n let selectize = $(select).selectize(opts)[0].selectize;\n\n let ctHandle = new FilterHandle(data.group);\n\n let lastKnownKeys;\n selectize.on(\"change\", function() {\n if (selectize.items.length === 0) {\n lastKnownKeys = null;\n ctHandle.clear();\n } else {\n let keys = {};\n selectize.items.forEach(function(group) {\n data.map[group].forEach(function(key) {\n keys[key] = true;\n });\n });\n let keyArray = Object.keys(keys);\n keyArray.sort();\n lastKnownKeys = keyArray;\n ctHandle.set(keyArray);\n }\n });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n","import * as input from \"./input\";\nimport { FilterHandle } from \"./filter\";\n\nlet $ = global.jQuery;\nlet strftime = global.strftime;\n\ninput.register({\n className: \"crosstalk-input-slider\",\n\n factory: function(el, data) {\n /*\n * map: {\"groupA\": [\"keyA\", \"keyB\", ...], ...}\n * group: \"ct-groupname\"\n */\n let ctHandle = new FilterHandle(data.group);\n\n let opts = {};\n let $el = $(el).find(\"input\");\n let dataType = $el.data(\"data-type\");\n let timeFormat = $el.data(\"time-format\");\n let round = $el.data(\"round\");\n let timeFormatter;\n\n // Set up formatting functions\n if (dataType === \"date\") {\n timeFormatter = strftime.utc();\n opts.prettify = function(num) {\n return timeFormatter(timeFormat, new Date(num));\n };\n\n } else if (dataType === \"datetime\") {\n let timezone = $el.data(\"timezone\");\n if (timezone)\n timeFormatter = strftime.timezone(timezone);\n else\n timeFormatter = strftime;\n\n opts.prettify = function(num) {\n return timeFormatter(timeFormat, new Date(num));\n };\n } else if (dataType === \"number\") {\n if (typeof round !== \"undefined\")\n opts.prettify = function(num) {\n let factor = Math.pow(10, round);\n return Math.round(num * factor) / factor;\n };\n }\n\n $el.ionRangeSlider(opts);\n\n function getValue() {\n let result = $el.data(\"ionRangeSlider\").result;\n\n // Function for converting numeric value from slider to appropriate type.\n let convert;\n let dataType = $el.data(\"data-type\");\n if (dataType === \"date\") {\n convert = function(val) {\n return formatDateUTC(new Date(+val));\n };\n } else if (dataType === \"datetime\") {\n convert = function(val) {\n // Convert ms to s\n return +val / 1000;\n };\n } else {\n convert = function(val) { return +val; };\n }\n\n if ($el.data(\"ionRangeSlider\").options.type === \"double\") {\n return [convert(result.from), convert(result.to)];\n } else {\n return convert(result.from);\n }\n }\n\n let lastKnownKeys = null;\n\n $el.on(\"change.crosstalkSliderInput\", function(event) {\n if (!$el.data(\"updating\") && !$el.data(\"animating\")) {\n let [from, to] = getValue();\n let keys = [];\n for (let i = 0; i < data.values.length; i++) {\n let val = data.values[i];\n if (val >= from && val <= to) {\n keys.push(data.keys[i]);\n }\n }\n keys.sort();\n ctHandle.set(keys);\n lastKnownKeys = keys;\n }\n });\n\n\n // let $el = $(el);\n // $el.on(\"change\", \"input[type=\"checkbox\"]\", function() {\n // let checked = $el.find(\"input[type=\"checkbox\"]:checked\");\n // if (checked.length === 0) {\n // ctHandle.clear();\n // } else {\n // let keys = {};\n // checked.each(function() {\n // data.map[this.value].forEach(function(key) {\n // keys[key] = true;\n // });\n // });\n // let keyArray = Object.keys(keys);\n // keyArray.sort();\n // ctHandle.set(keyArray);\n // }\n // });\n\n return {\n suspend: function() {\n ctHandle.clear();\n },\n resume: function() {\n if (lastKnownKeys)\n ctHandle.set(lastKnownKeys);\n }\n };\n }\n});\n\n\n// Convert a number to a string with leading zeros\nfunction padZeros(n, digits) {\n let str = n.toString();\n while (str.length < digits)\n str = \"0\" + str;\n return str;\n}\n\n// Given a Date object, return a string in yyyy-mm-dd format, using the\n// UTC date. This may be a day off from the date in the local time zone.\nfunction formatDateUTC(date) {\n if (date instanceof Date) {\n return date.getUTCFullYear() + \"-\" +\n padZeros(date.getUTCMonth()+1, 2) + \"-\" +\n padZeros(date.getUTCDate(), 2);\n\n } else {\n return null;\n }\n}\n","import Events from \"./events\";\nimport grp from \"./group\";\nimport * as util from \"./util\";\n\n/**\n * Use this class to read and write (and listen for changes to) the selection\n * for a Crosstalk group. This is intended to be used for linked brushing.\n *\n * If two (or more) `SelectionHandle` instances in the same webpage share the\n * same group name, they will share the same state. Setting the selection using\n * one `SelectionHandle` instance will result in the `value` property instantly\n * changing across the others, and `\"change\"` event listeners on all instances\n * (including the one that initiated the sending) will fire.\n *\n * @param {string} [group] - The name of the Crosstalk group, or if none,\n * null or undefined (or any other falsy value). This can be changed later\n * via the [SelectionHandle#setGroup](#setGroup) method.\n * @param {Object} [extraInfo] - An object whose properties will be copied to\n * the event object whenever an event is emitted.\n */\nexport class SelectionHandle {\n\n constructor(group = null, extraInfo = null) {\n this._eventRelay = new Events();\n this._emitter = new util.SubscriptionTracker(this._eventRelay);\n\n // Name of the group we're currently tracking, if any. Can change over time.\n this._group = null;\n // The Var we're currently tracking, if any. Can change over time.\n this._var = null;\n // The event handler subscription we currently have on var.on(\"change\").\n this._varOnChangeSub = null;\n\n this._extraInfo = util.extend({ sender: this }, extraInfo);\n\n this.setGroup(group);\n }\n\n /**\n * Changes the Crosstalk group membership of this SelectionHandle. The group\n * being switched away from (if any) will not have its selection value\n * modified as a result of calling `setGroup`, even if this handle was the\n * most recent handle to set the selection of the group.\n *\n * The group being switched to (if any) will also not have its selection value\n * modified as a result of calling `setGroup`. If you want to set the\n * selection value of the new group, call `set` explicitly.\n *\n * @param {string} group - The name of the Crosstalk group, or null (or\n * undefined) to clear the group.\n */\n setGroup(group) {\n // If group is unchanged, do nothing\n if (this._group === group)\n return;\n // Treat null, undefined, and other falsy values the same\n if (!this._group && !group)\n return;\n\n if (this._var) {\n this._var.off(\"change\", this._varOnChangeSub);\n this._var = null;\n this._varOnChangeSub = null;\n }\n\n this._group = group;\n\n if (group) {\n this._var = grp(group).var(\"selection\");\n let sub = this._var.on(\"change\", (e) => {\n this._eventRelay.trigger(\"change\", e, this);\n });\n this._varOnChangeSub = sub;\n }\n }\n\n /**\n * Retrieves the current selection for the group represented by this\n * `SelectionHandle`.\n *\n * - If no selection is active, then this value will be falsy.\n * - If a selection is active, but no data points are selected, then this\n * value will be an empty array.\n * - If a selection is active, and data points are selected, then the keys\n * of the selected data points will be present in the array.\n */\n get value() {\n return this._var ? this._var.get() : null;\n }\n\n /**\n * Combines the given `extraInfo` (if any) with the handle's default\n * `_extraInfo` (if any).\n * @private\n */\n _mergeExtraInfo(extraInfo) {\n // Important incidental effect: shallow clone is returned\n return util.extend({},\n this._extraInfo ? this._extraInfo : null,\n extraInfo ? extraInfo : null);\n }\n\n /**\n * Overwrites the current selection for the group, and raises the `\"change\"`\n * event among all of the group's '`SelectionHandle` instances (including\n * this one).\n *\n * @fires SelectionHandle#change\n * @param {string[]} selectedKeys - Falsy, empty array, or array of keys (see\n * {@link SelectionHandle#value}).\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any options that were\n * passed into the `SelectionHandle` constructor).\n */\n set(selectedKeys, extraInfo) {\n if (this._var)\n this._var.set(selectedKeys, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * Overwrites the current selection for the group, and raises the `\"change\"`\n * event among all of the group's '`SelectionHandle` instances (including\n * this one).\n *\n * @fires SelectionHandle#change\n * @param {Object} [extraInfo] - Extra properties to be included on the event\n * object that's passed to listeners (in addition to any that were passed\n * into the `SelectionHandle` constructor).\n */\n clear(extraInfo) {\n if (this._var)\n this.set(void 0, this._mergeExtraInfo(extraInfo));\n }\n\n /**\n * Subscribes to events on this `SelectionHandle`.\n *\n * @param {string} eventType - Indicates the type of events to listen to.\n * Currently, only `\"change\"` is supported.\n * @param {SelectionHandle~listener} listener - The callback function that\n * will be invoked when the event occurs.\n * @return {string} - A token to pass to {@link SelectionHandle#off} to cancel\n * this subscription.\n */\n on(eventType, listener) {\n return this._emitter.on(eventType, listener);\n }\n\n /**\n * Cancels event subscriptions created by {@link SelectionHandle#on}.\n *\n * @param {string} eventType - The type of event to unsubscribe.\n * @param {string|SelectionHandle~listener} listener - Either the callback\n * function previously passed into {@link SelectionHandle#on}, or the\n * string that was returned from {@link SelectionHandle#on}.\n */\n off(eventType, listener) {\n return this._emitter.off(eventType, listener);\n }\n\n /**\n * Shuts down the `SelectionHandle` object.\n *\n * Removes all event listeners that were added through this handle.\n */\n close() {\n this._emitter.removeAllListeners();\n this.setGroup(null);\n }\n}\n\n/**\n * @callback SelectionHandle~listener\n * @param {Object} event - An object containing details of the event. For\n * `\"change\"` events, this includes the properties `value` (the new\n * value of the selection, or `undefined` if no selection is active),\n * `oldValue` (the previous value of the selection), and `sender` (the\n * `SelectionHandle` instance that made the change).\n */\n\n/**\n * @event SelectionHandle#change\n * @type {object}\n * @property {object} value - The new value of the selection, or `undefined`\n * if no selection is active.\n * @property {object} oldValue - The previous value of the selection.\n * @property {SelectionHandle} sender - The `SelectionHandle` instance that\n * changed the value.\n */\n","export function extend(target, ...sources) {\n for (let i = 0; i < sources.length; i++) {\n let src = sources[i];\n if (typeof(src) === \"undefined\" || src === null)\n continue;\n\n for (let key in src) {\n if (src.hasOwnProperty(key)) {\n target[key] = src[key];\n }\n }\n }\n return target;\n}\n\nexport function checkSorted(list) {\n for (let i = 1; i < list.length; i++) {\n if (list[i] <= list[i-1]) {\n throw new Error(\"List is not sorted or contains duplicate\");\n }\n }\n}\n\nexport function diffSortedLists(a, b) {\n let i_a = 0;\n let i_b = 0;\n\n if (!a) a = [];\n if (!b) b = [];\n\n let a_only = [];\n let b_only = [];\n\n checkSorted(a);\n checkSorted(b);\n\n while (i_a < a.length && i_b < b.length) {\n if (a[i_a] === b[i_b]) {\n i_a++;\n i_b++;\n } else if (a[i_a] < b[i_b]) {\n a_only.push(a[i_a++]);\n } else {\n b_only.push(b[i_b++]);\n }\n }\n\n if (i_a < a.length)\n a_only = a_only.concat(a.slice(i_a));\n if (i_b < b.length)\n b_only = b_only.concat(b.slice(i_b));\n return {\n removed: a_only,\n added: b_only\n };\n}\n\n// Convert from wide: { colA: [1,2,3], colB: [4,5,6], ... }\n// to long: [ {colA: 1, colB: 4}, {colA: 2, colB: 5}, ... ]\nexport function dataframeToD3(df) {\n let names = [];\n let length;\n for (let name in df) {\n if (df.hasOwnProperty(name))\n names.push(name);\n if (typeof(df[name]) !== \"object\" || typeof(df[name].length) === \"undefined\") {\n throw new Error(\"All fields must be arrays\");\n } else if (typeof(length) !== \"undefined\" && length !== df[name].length) {\n throw new Error(\"All fields must be arrays of the same length\");\n }\n length = df[name].length;\n }\n let results = [];\n let item;\n for (let row = 0; row < length; row++) {\n item = {};\n for (let col = 0; col < names.length; col++) {\n item[names[col]] = df[names[col]][row];\n }\n results.push(item);\n }\n return results;\n}\n\n/**\n * Keeps track of all event listener additions/removals and lets all active\n * listeners be removed with a single operation.\n *\n * @private\n */\nexport class SubscriptionTracker {\n constructor(emitter) {\n this._emitter = emitter;\n this._subs = {};\n }\n\n on(eventType, listener) {\n let sub = this._emitter.on(eventType, listener);\n this._subs[sub] = eventType;\n return sub;\n }\n\n off(eventType, listener) {\n let sub = this._emitter.off(eventType, listener);\n if (sub) {\n delete this._subs[sub];\n }\n return sub;\n }\n\n removeAllListeners() {\n let current_subs = this._subs;\n this._subs = {};\n Object.keys(current_subs).forEach((sub) => {\n this._emitter.off(current_subs[sub], sub);\n });\n }\n}\n","import Events from \"./events\";\n\nexport default class Var {\n constructor(group, name, /*optional*/ value) {\n this._group = group;\n this._name = name;\n this._value = value;\n this._events = new Events();\n }\n\n get() {\n return this._value;\n }\n\n set(value, /*optional*/ event) {\n if (this._value === value) {\n // Do nothing; the value hasn't changed\n return;\n }\n let oldValue = this._value;\n this._value = value;\n // Alert JavaScript listeners that the value has changed\n let evt = {};\n if (event && typeof(event) === \"object\") {\n for (let k in event) {\n if (event.hasOwnProperty(k))\n evt[k] = event[k];\n }\n }\n evt.oldValue = oldValue;\n evt.value = value;\n this._events.trigger(\"change\", evt, this);\n\n // TODO: Make this extensible, to let arbitrary back-ends know that\n // something has changed\n if (global.Shiny && global.Shiny.onInputChange) {\n global.Shiny.onInputChange(\n \".clientValue-\" +\n (this._group.name !== null ? this._group.name + \"-\" : \"\") +\n this._name,\n typeof(value) === \"undefined\" ? null : value\n );\n }\n }\n\n on(eventType, listener) {\n return this._events.on(eventType, listener);\n }\n\n off(eventType, listener) {\n return this._events.off(eventType, listener);\n }\n}\n"]} \ No newline at end of file diff --git a/docs/site_libs/crosstalk-1.2.1/scss/crosstalk.scss b/docs/site_libs/crosstalk-1.2.1/scss/crosstalk.scss new file mode 100644 index 00000000..35665616 --- /dev/null +++ b/docs/site_libs/crosstalk-1.2.1/scss/crosstalk.scss @@ -0,0 +1,75 @@ +/* Adjust margins outwards, so column contents line up with the edges of the + parent of container-fluid. */ +.container-fluid.crosstalk-bscols { + margin-left: -30px; + margin-right: -30px; + white-space: normal; +} + +/* But don't adjust the margins outwards if we're directly under the body, + i.e. we were the top-level of something at the console. */ +body > .container-fluid.crosstalk-bscols { + margin-left: auto; + margin-right: auto; +} + +.crosstalk-input-checkboxgroup .crosstalk-options-group .crosstalk-options-column { + display: inline-block; + padding-right: 12px; + vertical-align: top; +} + +@media only screen and (max-width:480px) { + .crosstalk-input-checkboxgroup .crosstalk-options-group .crosstalk-options-column { + display: block; + padding-right: inherit; + } +} + +/* Relevant BS3 styles to make filter_checkbox() look reasonable without Bootstrap */ +.crosstalk-input { + margin-bottom: 15px; /* a la .form-group */ + .control-label { + margin-bottom: 0; + vertical-align: middle; + } + input[type="checkbox"] { + margin: 4px 0 0; + margin-top: 1px; + line-height: normal; + } + .checkbox { + position: relative; + display: block; + margin-top: 10px; + margin-bottom: 10px; + } + .checkbox > label{ + padding-left: 20px; + margin-bottom: 0; + font-weight: 400; + cursor: pointer; + } + .checkbox input[type="checkbox"], + .checkbox-inline input[type="checkbox"] { + position: absolute; + margin-top: 2px; + margin-left: -20px; + } + .checkbox + .checkbox { + margin-top: -5px; + } + .checkbox-inline { + position: relative; + display: inline-block; + padding-left: 20px; + margin-bottom: 0; + font-weight: 400; + vertical-align: middle; + cursor: pointer; + } + .checkbox-inline + .checkbox-inline { + margin-top: 0; + margin-left: 10px; + } +} diff --git a/docs/site_libs/datatables-binding-0.33/datatables.js b/docs/site_libs/datatables-binding-0.33/datatables.js new file mode 100644 index 00000000..765b53cb --- /dev/null +++ b/docs/site_libs/datatables-binding-0.33/datatables.js @@ -0,0 +1,1539 @@ +(function() { + +// some helper functions: using a global object DTWidget so that it can be used +// in JS() code, e.g. datatable(options = list(foo = JS('code'))); unlike R's +// dynamic scoping, when 'code' is eval'ed, JavaScript does not know objects +// from the "parent frame", e.g. JS('DTWidget') will not work unless it was made +// a global object +var DTWidget = {}; + +// 123456666.7890 -> 123,456,666.7890 +var markInterval = function(d, digits, interval, mark, decMark, precision) { + x = precision ? d.toPrecision(digits) : d.toFixed(digits); + if (!/^-?[\d.]+$/.test(x)) return x; + var xv = x.split('.'); + if (xv.length > 2) return x; // should have at most one decimal point + xv[0] = xv[0].replace(new RegExp('\\B(?=(\\d{' + interval + '})+(?!\\d))', 'g'), mark); + return xv.join(decMark); +}; + +DTWidget.formatCurrency = function(data, currency, digits, interval, mark, decMark, before, zeroPrint) { + var d = parseFloat(data); + if (isNaN(d)) return ''; + if (zeroPrint !== null && d === 0.0) return zeroPrint; + var res = markInterval(d, digits, interval, mark, decMark); + res = before ? (/^-/.test(res) ? '-' + currency + res.replace(/^-/, '') : currency + res) : + res + currency; + return res; +}; + +DTWidget.formatString = function(data, prefix, suffix) { + var d = data; + if (d === null) return ''; + return prefix + d + suffix; +}; + +DTWidget.formatPercentage = function(data, digits, interval, mark, decMark, zeroPrint) { + var d = parseFloat(data); + if (isNaN(d)) return ''; + if (zeroPrint !== null && d === 0.0) return zeroPrint; + return markInterval(d * 100, digits, interval, mark, decMark) + '%'; +}; + +DTWidget.formatRound = function(data, digits, interval, mark, decMark, zeroPrint) { + var d = parseFloat(data); + if (isNaN(d)) return ''; + if (zeroPrint !== null && d === 0.0) return zeroPrint; + return markInterval(d, digits, interval, mark, decMark); +}; + +DTWidget.formatSignif = function(data, digits, interval, mark, decMark, zeroPrint) { + var d = parseFloat(data); + if (isNaN(d)) return ''; + if (zeroPrint !== null && d === 0.0) return zeroPrint; + return markInterval(d, digits, interval, mark, decMark, true); +}; + +DTWidget.formatDate = function(data, method, params) { + var d = data; + if (d === null) return ''; + // (new Date('2015-10-28')).toDateString() may return 2015-10-27 because the + // actual time created could be like 'Tue Oct 27 2015 19:00:00 GMT-0500 (CDT)', + // i.e. the date-only string is treated as UTC time instead of local time + if ((method === 'toDateString' || method === 'toLocaleDateString') && /^\d{4,}\D\d{2}\D\d{2}$/.test(d)) { + d = d.split(/\D/); + d = new Date(d[0], d[1] - 1, d[2]); + } else { + d = new Date(d); + } + return d[method].apply(d, params); +}; + +window.DTWidget = DTWidget; + +// A helper function to update the properties of existing filters +var setFilterProps = function(td, props) { + // Update enabled/disabled state + var $input = $(td).find('input').first(); + var searchable = $input.data('searchable'); + $input.prop('disabled', !searchable || props.disabled); + + // Based on the filter type, set its new values + var type = td.getAttribute('data-type'); + if (['factor', 'logical'].includes(type)) { + // Reformat the new dropdown options for use with selectize + var new_vals = props.params.options.map(function(item) { + return { text: item, value: item }; + }); + + // Find the selectize object + var dropdown = $(td).find('.selectized').eq(0)[0].selectize; + + // Note the current values + var old_vals = dropdown.getValue(); + + // Remove the existing values + dropdown.clearOptions(); + + // Add the new options + dropdown.addOption(new_vals); + + // Preserve the existing values + dropdown.setValue(old_vals); + + } else if (['number', 'integer', 'date', 'time'].includes(type)) { + // Apply internal scaling to new limits. Updating scale not yet implemented. + var slider = $(td).find('.noUi-target').eq(0); + var scale = Math.pow(10, Math.max(0, +slider.data('scale') || 0)); + var new_vals = [props.params.min * scale, props.params.max * scale]; + + // Note what the new limits will be just for this filter + var new_lims = new_vals.slice(); + + // Determine the current values and limits + var old_vals = slider.val().map(Number); + var old_lims = slider.noUiSlider('options').range; + old_lims = [old_lims.min, old_lims.max]; + + // Preserve the current values if filters have been applied; otherwise, apply no filtering + if (old_vals[0] != old_lims[0]) { + new_vals[0] = Math.max(old_vals[0], new_vals[0]); + } + + if (old_vals[1] != old_lims[1]) { + new_vals[1] = Math.min(old_vals[1], new_vals[1]); + } + + // Update the endpoints of the slider + slider.noUiSlider({ + start: new_vals, + range: {'min': new_lims[0], 'max': new_lims[1]} + }, true); + } +}; + +var transposeArray2D = function(a) { + return a.length === 0 ? a : HTMLWidgets.transposeArray2D(a); +}; + +var crosstalkPluginsInstalled = false; + +function maybeInstallCrosstalkPlugins() { + if (crosstalkPluginsInstalled) + return; + crosstalkPluginsInstalled = true; + + $.fn.dataTable.ext.afnFiltering.push( + function(oSettings, aData, iDataIndex) { + var ctfilter = oSettings.nTable.ctfilter; + if (ctfilter && !ctfilter[iDataIndex]) + return false; + + var ctselect = oSettings.nTable.ctselect; + if (ctselect && !ctselect[iDataIndex]) + return false; + + return true; + } + ); +} + +HTMLWidgets.widget({ + name: "datatables", + type: "output", + renderOnNullValue: true, + initialize: function(el, width, height) { + // in order that the type=number inputs return a number + $.valHooks.number = { + get: function(el) { + var value = parseFloat(el.value); + return isNaN(value) ? "" : value; + } + }; + $(el).html(' '); + return { + data: null, + ctfilterHandle: new crosstalk.FilterHandle(), + ctfilterSubscription: null, + ctselectHandle: new crosstalk.SelectionHandle(), + ctselectSubscription: null + }; + }, + renderValue: function(el, data, instance) { + if (el.offsetWidth === 0 || el.offsetHeight === 0) { + instance.data = data; + return; + } + instance.data = null; + var $el = $(el); + $el.empty(); + + if (data === null) { + $el.append(' '); + // clear previous Shiny inputs (if any) + for (var i in instance.clearInputs) instance.clearInputs[i](); + instance.clearInputs = {}; + return; + } + + var crosstalkOptions = data.crosstalkOptions; + if (!crosstalkOptions) crosstalkOptions = { + 'key': null, 'group': null + }; + if (crosstalkOptions.group) { + maybeInstallCrosstalkPlugins(); + instance.ctfilterHandle.setGroup(crosstalkOptions.group); + instance.ctselectHandle.setGroup(crosstalkOptions.group); + } + + // if we are in the viewer then we always want to fillContainer and + // and autoHideNavigation (unless the user has explicitly set these) + if (window.HTMLWidgets.viewerMode) { + if (!data.hasOwnProperty("fillContainer")) + data.fillContainer = true; + if (!data.hasOwnProperty("autoHideNavigation")) + data.autoHideNavigation = true; + } + + // propagate fillContainer to instance (so we have it in resize) + instance.fillContainer = data.fillContainer; + + var cells = data.data; + + if (cells instanceof Array) cells = transposeArray2D(cells); + + $el.append(data.container); + var $table = $el.find('table'); + if (data.class) $table.addClass(data.class); + if (data.caption) $table.prepend(data.caption); + + if (!data.selection) data.selection = { + mode: 'none', selected: null, target: 'row', selectable: null + }; + if (HTMLWidgets.shinyMode && data.selection.mode !== 'none' && + data.selection.target === 'row+column') { + if ($table.children('tfoot').length === 0) { + $table.append($('')); + $table.find('thead tr').clone().appendTo($table.find('tfoot')); + } + } + + // column filters + var filterRow; + switch (data.filter) { + case 'top': + $table.children('thead').append(data.filterHTML); + filterRow = $table.find('thead tr:last td'); + break; + case 'bottom': + if ($table.children('tfoot').length === 0) { + $table.append($('')); + } + $table.children('tfoot').prepend(data.filterHTML); + filterRow = $table.find('tfoot tr:first td'); + break; + } + + var options = { searchDelay: 1000 }; + if (cells !== null) $.extend(options, { + data: cells + }); + + // options for fillContainer + var bootstrapActive = typeof($.fn.popover) != 'undefined'; + if (instance.fillContainer) { + + // force scrollX/scrollY and turn off autoWidth + options.scrollX = true; + options.scrollY = "100px"; // can be any value, we'll adjust below + + // if we aren't paginating then move around the info/filter controls + // to save space at the bottom and rephrase the info callback + if (data.options.paging === false) { + + // we know how to do this cleanly for bootstrap, not so much + // for other themes/layouts + if (bootstrapActive) { + options.dom = "<'row'<'col-sm-4'i><'col-sm-8'f>>" + + "<'row'<'col-sm-12'tr>>"; + } + + options.fnInfoCallback = function(oSettings, iStart, iEnd, + iMax, iTotal, sPre) { + return Number(iTotal).toLocaleString() + " records"; + }; + } + } + + // auto hide navigation if requested + // Note, this only works on client-side processing mode as on server-side, + // cells (data.data) is null; In addition, we require the pageLength option + // being provided explicitly to enable this. Despite we may be able to deduce + // the default value of pageLength, it may complicate things so we'd rather + // put this responsiblity to users and warn them on the R side. + if (data.autoHideNavigation === true && data.options.paging !== false) { + // strip all nav if length >= cells + if ((cells instanceof Array) && data.options.pageLength >= cells.length) + options.dom = bootstrapActive ? "<'row'<'col-sm-12'tr>>" : "t"; + // alternatively lean things out for flexdashboard mobile portrait + else if (bootstrapActive && window.FlexDashboard && window.FlexDashboard.isMobilePhone()) + options.dom = "<'row'<'col-sm-12'f>>" + + "<'row'<'col-sm-12'tr>>" + + "<'row'<'col-sm-12'p>>"; + } + + $.extend(true, options, data.options || {}); + + var searchCols = options.searchCols; + if (searchCols) { + searchCols = searchCols.map(function(x) { + return x === null ? '' : x.search; + }); + // FIXME: this means I don't respect the escapeRegex setting + delete options.searchCols; + } + + // server-side processing? + var server = options.serverSide === true; + + // use the dataSrc function to pre-process JSON data returned from R + var DT_rows_all = [], DT_rows_current = []; + if (server && HTMLWidgets.shinyMode && typeof options.ajax === 'object' && + /^session\/[\da-z]+\/dataobj/.test(options.ajax.url) && !options.ajax.dataSrc) { + options.ajax.dataSrc = function(json) { + DT_rows_all = $.makeArray(json.DT_rows_all); + DT_rows_current = $.makeArray(json.DT_rows_current); + var data = json.data; + if (!colReorderEnabled()) return data; + var table = $table.DataTable(), order = table.colReorder.order(), flag = true, i, j, row; + for (i = 0; i < order.length; ++i) if (order[i] !== i) flag = false; + if (flag) return data; + for (i = 0; i < data.length; ++i) { + row = data[i].slice(); + for (j = 0; j < order.length; ++j) data[i][j] = row[order[j]]; + } + return data; + }; + } + + var thiz = this; + if (instance.fillContainer) $table.on('init.dt', function(e) { + thiz.fillAvailableHeight(el, $(el).innerHeight()); + }); + // If the page contains serveral datatables and one of which enables colReorder, + // the table.colReorder.order() function will exist but throws error when called. + // So it seems like the only way to know if colReorder is enabled or not is to + // check the options. + var colReorderEnabled = function() { return "colReorder" in options; }; + var table = $table.DataTable(options); + $el.data('datatable', table); + + if ('rowGroup' in options) { + // Maintain RowGroup dataSrc when columns are reordered (#1109) + table.on('column-reorder', function(e, settings, details) { + var oldDataSrc = table.rowGroup().dataSrc(); + var newDataSrc = details.mapping[oldDataSrc]; + table.rowGroup().dataSrc(newDataSrc); + }); + } + + // Unregister previous Crosstalk event subscriptions, if they exist + if (instance.ctfilterSubscription) { + instance.ctfilterHandle.off("change", instance.ctfilterSubscription); + instance.ctfilterSubscription = null; + } + if (instance.ctselectSubscription) { + instance.ctselectHandle.off("change", instance.ctselectSubscription); + instance.ctselectSubscription = null; + } + + if (!crosstalkOptions.group) { + $table[0].ctfilter = null; + $table[0].ctselect = null; + } else { + var key = crosstalkOptions.key; + function keysToMatches(keys) { + if (!keys) { + return null; + } else { + var selectedKeys = {}; + for (var i = 0; i < keys.length; i++) { + selectedKeys[keys[i]] = true; + } + var matches = {}; + for (var j = 0; j < key.length; j++) { + if (selectedKeys[key[j]]) + matches[j] = true; + } + return matches; + } + } + + function applyCrosstalkFilter(e) { + $table[0].ctfilter = keysToMatches(e.value); + table.draw(); + } + instance.ctfilterSubscription = instance.ctfilterHandle.on("change", applyCrosstalkFilter); + applyCrosstalkFilter({value: instance.ctfilterHandle.filteredKeys}); + + function applyCrosstalkSelection(e) { + if (e.sender !== instance.ctselectHandle) { + table + .rows('.' + selClass, {search: 'applied'}) + .nodes() + .to$() + .removeClass(selClass); + if (selectedRows) + changeInput('rows_selected', selectedRows(), void 0, true); + } + + if (e.sender !== instance.ctselectHandle && e.value && e.value.length) { + var matches = keysToMatches(e.value); + + // persistent selection with plotly (& leaflet) + var ctOpts = crosstalk.var("plotlyCrosstalkOpts").get() || {}; + if (ctOpts.persistent === true) { + var matches = $.extend(matches, $table[0].ctselect); + } + + $table[0].ctselect = matches; + table.draw(); + } else { + if ($table[0].ctselect) { + $table[0].ctselect = null; + table.draw(); + } + } + } + instance.ctselectSubscription = instance.ctselectHandle.on("change", applyCrosstalkSelection); + // TODO: This next line doesn't seem to work when renderDataTable is used + applyCrosstalkSelection({value: instance.ctselectHandle.value}); + } + + var inArray = function(val, array) { + return $.inArray(val, $.makeArray(array)) > -1; + }; + + // search the i-th column + var searchColumn = function(i, value) { + var regex = false, ci = true; + if (options.search) { + regex = options.search.regex, + ci = options.search.caseInsensitive !== false; + } + // need to transpose the column index when colReorder is enabled + if (table.colReorder) i = table.colReorder.transpose(i); + return table.column(i).search(value, regex, !regex, ci); + }; + + if (data.filter !== 'none') { + if (!data.hasOwnProperty('filterSettings')) data.filterSettings = {}; + + filterRow.each(function(i, td) { + + var $td = $(td), type = $td.data('type'), filter; + var $input = $td.children('div').first().children('input'); + var disabled = $input.prop('disabled'); + var searchable = table.settings()[0].aoColumns[i].bSearchable; + $input.prop('disabled', !searchable || disabled); + $input.data('searchable', searchable); // for updating later + $input.on('input blur', function() { + $input.next('span').toggle(Boolean($input.val())); + }); + // Bootstrap sets pointer-events to none and we won't be able to click + // the clear button + $input.next('span').css('pointer-events', 'auto').hide().click(function() { + $(this).hide().prev('input').val('').trigger('input').focus(); + }); + var searchCol; // search string for this column + if (searchCols && searchCols[i]) { + searchCol = searchCols[i]; + $input.val(searchCol).trigger('input'); + } + var $x = $td.children('div').last(); + + // remove the overflow: hidden attribute of the scrollHead + // (otherwise the scrolling table body obscures the filters) + // The workaround and the discussion from + // https://github.com/rstudio/DT/issues/554#issuecomment-518007347 + // Otherwise the filter selection will not be anchored to the values + // when the columns number is many and scrollX is enabled. + var scrollHead = $(el).find('.dataTables_scrollHead,.dataTables_scrollFoot'); + var cssOverflowHead = scrollHead.css('overflow'); + var scrollBody = $(el).find('.dataTables_scrollBody'); + var cssOverflowBody = scrollBody.css('overflow'); + var scrollTable = $(el).find('.dataTables_scroll'); + var cssOverflowTable = scrollTable.css('overflow'); + if (cssOverflowHead === 'hidden') { + $x.on('show hide', function(e) { + if (e.type === 'show') { + scrollHead.css('overflow', 'visible'); + scrollBody.css('overflow', 'visible'); + scrollTable.css('overflow-x', 'scroll'); + } else { + scrollHead.css('overflow', cssOverflowHead); + scrollBody.css('overflow', cssOverflowBody); + scrollTable.css('overflow-x', cssOverflowTable); + } + }); + $x.css('z-index', 25); + } + + if (inArray(type, ['factor', 'logical'])) { + $input.on({ + click: function() { + $input.parent().hide(); $x.show().trigger('show'); filter[0].selectize.focus(); + }, + input: function() { + var v1 = JSON.stringify(filter[0].selectize.getValue()), v2 = $input.val(); + if (v1 === '[]') v1 = ''; + if (v1 !== v2) filter[0].selectize.setValue(v2 === '' ? [] : JSON.parse(v2)); + } + }); + var $input2 = $x.children('select'); + filter = $input2.selectize($.extend({ + options: $input2.data('options').map(function(v, i) { + return ({text: v, value: v}); + }), + plugins: ['remove_button'], + hideSelected: true, + onChange: function(value) { + if (value === null) value = []; // compatibility with jQuery 3.0 + $input.val(value.length ? JSON.stringify(value) : ''); + if (value.length) $input.trigger('input'); + $input.attr('title', $input.val()); + if (server) { + searchColumn(i, value.length ? JSON.stringify(value) : '').draw(); + return; + } + // turn off filter if nothing selected + $td.data('filter', value.length > 0); + table.draw(); // redraw table, and filters will be applied + } + }, data.filterSettings.select)); + filter[0].selectize.on('blur', function() { + $x.hide().trigger('hide'); $input.parent().show(); $input.trigger('blur'); + }); + filter.next('div').css('margin-bottom', 'auto'); + } else if (type === 'character') { + var fun = function() { + searchColumn(i, $input.val()).draw(); + }; + // throttle searching for server-side processing + var throttledFun = $.fn.dataTable.util.throttle(fun, options.searchDelay); + $input.on('input', function(e, immediate) { + // always bypass throttling when immediate = true (via the updateSearch method) + (immediate || !server) ? fun() : throttledFun(); + }); + } else if (inArray(type, ['number', 'integer', 'date', 'time'])) { + var $x0 = $x; + $x = $x0.children('div').first(); + $x0.css({ + 'background-color': '#fff', + 'border': '1px #ddd solid', + 'border-radius': '4px', + 'padding': data.vertical ? '35px 20px': '20px 20px 10px 20px' + }); + var $spans = $x0.children('span').css({ + 'margin-top': data.vertical ? '0' : '10px', + 'white-space': 'nowrap' + }); + var $span1 = $spans.first(), $span2 = $spans.last(); + var r1 = +$x.data('min'), r2 = +$x.data('max'); + // when the numbers are too small or have many decimal places, the + // slider may have numeric precision problems (#150) + var scale = Math.pow(10, Math.max(0, +$x.data('scale') || 0)); + r1 = Math.round(r1 * scale); r2 = Math.round(r2 * scale); + var scaleBack = function(x, scale) { + if (scale === 1) return x; + var d = Math.round(Math.log(scale) / Math.log(10)); + // to avoid problems like 3.423/100 -> 0.034230000000000003 + return (x / scale).toFixed(d); + }; + var slider_min = function() { + return filter.noUiSlider('options').range.min; + }; + var slider_max = function() { + return filter.noUiSlider('options').range.max; + }; + $input.on({ + focus: function() { + $x0.show().trigger('show'); + // first, make sure the slider div leaves at least 20px between + // the two (slider value) span's + $x0.width(Math.max(160, $span1.outerWidth() + $span2.outerWidth() + 20)); + // then, if the input is really wide or slider is vertical, + // make the slider the same width as the input + if ($x0.outerWidth() < $input.outerWidth() || data.vertical) { + $x0.outerWidth($input.outerWidth()); + } + // make sure the slider div does not reach beyond the right margin + if ($(window).width() < $x0.offset().left + $x0.width()) { + $x0.offset({ + 'left': $input.offset().left + $input.outerWidth() - $x0.outerWidth() + }); + } + }, + blur: function() { + $x0.hide().trigger('hide'); + }, + input: function() { + if ($input.val() === '') filter.val([slider_min(), slider_max()]); + }, + change: function() { + var v = $input.val().replace(/\s/g, ''); + if (v === '') return; + v = v.split('...'); + if (v.length !== 2) { + $input.parent().addClass('has-error'); + return; + } + if (v[0] === '') v[0] = slider_min(); + if (v[1] === '') v[1] = slider_max(); + $input.parent().removeClass('has-error'); + // treat date as UTC time at midnight + var strTime = function(x) { + var s = type === 'date' ? 'T00:00:00Z' : ''; + var t = new Date(x + s).getTime(); + // add 10 minutes to date since it does not hurt the date, and + // it helps avoid the tricky floating point arithmetic problems, + // e.g. sometimes the date may be a few milliseconds earlier + // than the midnight due to precision problems in noUiSlider + return type === 'date' ? t + 3600000 : t; + }; + if (inArray(type, ['date', 'time'])) { + v[0] = strTime(v[0]); + v[1] = strTime(v[1]); + } + if (v[0] != slider_min()) v[0] *= scale; + if (v[1] != slider_max()) v[1] *= scale; + filter.val(v); + } + }); + var formatDate = function(d) { + d = scaleBack(d, scale); + if (type === 'number') return d; + if (type === 'integer') return parseInt(d); + var x = new Date(+d); + if (type === 'date') { + var pad0 = function(x) { + return ('0' + x).substr(-2, 2); + }; + return x.getUTCFullYear() + '-' + pad0(1 + x.getUTCMonth()) + + '-' + pad0(x.getUTCDate()); + } else { + return x.toISOString(); + } + }; + var opts = type === 'date' ? { step: 60 * 60 * 1000 } : + type === 'integer' ? { step: 1 } : {}; + + opts.orientation = data.vertical ? 'vertical': 'horizontal'; + opts.direction = data.vertical ? 'rtl': 'ltr'; + + filter = $x.noUiSlider($.extend({ + start: [r1, r2], + range: {min: r1, max: r2}, + connect: true + }, opts, data.filterSettings.slider)); + if (scale > 1) (function() { + var t1 = r1, t2 = r2; + var val = filter.val(); + while (val[0] > r1 || val[1] < r2) { + if (val[0] > r1) { + t1 -= val[0] - r1; + } + if (val[1] < r2) { + t2 += r2 - val[1]; + } + filter = $x.noUiSlider($.extend({ + start: [t1, t2], + range: {min: t1, max: t2}, + connect: true + }, opts, data.filterSettings.slider), true); + val = filter.val(); + } + r1 = t1; r2 = t2; + })(); + // format with active column renderer, if defined + var colDef = data.options.columnDefs.find(function(def) { + return (def.targets === i || inArray(i, def.targets)) && 'render' in def; + }); + var updateSliderText = function(v1, v2) { + // we only know how to use function renderers + if (colDef && typeof colDef.render === 'function') { + var restore = function(v) { + v = scaleBack(v, scale); + return inArray(type, ['date', 'time']) ? new Date(+v) : v; + } + $span1.text(colDef.render(restore(v1), 'display')); + $span2.text(colDef.render(restore(v2), 'display')); + } else { + $span1.text(formatDate(v1)); + $span2.text(formatDate(v2)); + } + }; + updateSliderText(r1, r2); + var updateSlider = function(e) { + var val = filter.val(); + // turn off filter if in full range + $td.data('filter', val[0] > slider_min() || val[1] < slider_max()); + var v1 = formatDate(val[0]), v2 = formatDate(val[1]), ival; + if ($td.data('filter')) { + ival = v1 + ' ... ' + v2; + $input.attr('title', ival).val(ival).trigger('input'); + } else { + $input.attr('title', '').val(''); + } + updateSliderText(val[0], val[1]); + if (e.type === 'slide') return; // no searching when sliding only + if (server) { + searchColumn(i, $td.data('filter') ? ival : '').draw(); + return; + } + table.draw(); + }; + filter.on({ + set: updateSlider, + slide: updateSlider + }); + } + + // server-side processing will be handled by R (or whatever server + // language you use); the following code is only needed for client-side + // processing + if (server) { + // if a search string has been pre-set, search now + if (searchCol) $input.trigger('input').trigger('change'); + return; + } + + var customFilter = function(settings, data, dataIndex) { + // there is no way to attach a search function to a specific table, + // and we need to make sure a global search function is not applied to + // all tables (i.e. a range filter in a previous table should not be + // applied to the current table); we use the settings object to + // determine if we want to perform searching on the current table, + // since settings.sTableId will be different to different tables + if (table.settings()[0] !== settings) return true; + // no filter on this column or no need to filter this column + if (typeof filter === 'undefined' || !$td.data('filter')) return true; + + var r = filter.val(), v, r0, r1; + var i_data = function(i) { + if (!colReorderEnabled()) return i; + var order = table.colReorder.order(), k; + for (k = 0; k < order.length; ++k) if (order[k] === i) return k; + return i; // in theory it will never be here... + } + v = data[i_data(i)]; + if (type === 'number' || type === 'integer') { + v = parseFloat(v); + // how to handle NaN? currently exclude these rows + if (isNaN(v)) return(false); + r0 = parseFloat(scaleBack(r[0], scale)) + r1 = parseFloat(scaleBack(r[1], scale)); + if (v >= r0 && v <= r1) return true; + } else if (type === 'date' || type === 'time') { + v = new Date(v); + r0 = new Date(r[0] / scale); r1 = new Date(r[1] / scale); + if (v >= r0 && v <= r1) return true; + } else if (type === 'factor') { + if (r.length === 0 || inArray(v, r)) return true; + } else if (type === 'logical') { + if (r.length === 0) return true; + if (inArray(v === '' ? 'na' : v, r)) return true; + } + return false; + }; + + $.fn.dataTable.ext.search.push(customFilter); + + // search for the preset search strings if it is non-empty + if (searchCol) $input.trigger('input').trigger('change'); + + }); + + } + + // highlight search keywords + var highlight = function() { + var body = $(table.table().body()); + // removing the old highlighting first + body.unhighlight(); + + // don't highlight the "not found" row, so we get the rows using the api + if (table.rows({ filter: 'applied' }).data().length === 0) return; + // highlight global search keywords + body.highlight($.trim(table.search()).split(/\s+/)); + // then highlight keywords from individual column filters + if (filterRow) filterRow.each(function(i, td) { + var $td = $(td), type = $td.data('type'); + if (type !== 'character') return; + var $input = $td.children('div').first().children('input'); + var column = table.column(i).nodes().to$(), + val = $.trim($input.val()); + if (type !== 'character' || val === '') return; + column.highlight(val.split(/\s+/)); + }); + }; + + if (options.searchHighlight) { + table + .on('draw.dt.dth column-visibility.dt.dth column-reorder.dt.dth', highlight) + .on('destroy', function() { + // remove event handler + table.off('draw.dt.dth column-visibility.dt.dth column-reorder.dt.dth'); + }); + + // Set the option for escaping regex characters in our search string. This will be used + // for all future matching. + jQuery.fn.highlight.options.escapeRegex = (!options.search || !options.search.regex); + + // initial highlight for state saved conditions and initial states + highlight(); + } + + // run the callback function on the table instance + if (typeof data.callback === 'function') data.callback(table); + + // double click to edit the cell, row, column, or all cells + if (data.editable) table.on('dblclick.dt', 'tbody td', function(e) { + // only bring up the editor when the cell itself is dbclicked, and ignore + // other dbclick events bubbled up (e.g. from the ) + if (e.target !== this) return; + var target = [], immediate = false; + switch (data.editable.target) { + case 'cell': + target = [this]; + immediate = true; // edit will take effect immediately + break; + case 'row': + target = table.cells(table.cell(this).index().row, '*').nodes(); + break; + case 'column': + target = table.cells('*', table.cell(this).index().column).nodes(); + break; + case 'all': + target = table.cells().nodes(); + break; + default: + throw 'The editable parameter must be "cell", "row", "column", or "all"'; + } + var disableCols = data.editable.disable ? data.editable.disable.columns : null; + var numericCols = data.editable.numeric; + var areaCols = data.editable.area; + var dateCols = data.editable.date; + for (var i = 0; i < target.length; i++) { + (function(cell, current) { + var $cell = $(cell), html = $cell.html(); + var _cell = table.cell(cell), value = _cell.data(), index = _cell.index().column; + var $input; + if (inArray(index, numericCols)) { + $input = $(''); + } else if (inArray(index, areaCols)) { + $input = $(''); + } else if (inArray(index, dateCols)) { + $input = $(''); + } else { + $input = $(''); + } + if (!immediate) { + $cell.data('input', $input).data('html', html); + $input.attr('title', 'Hit Ctrl+Enter to finish editing, or Esc to cancel'); + } + $input.val(value); + if (inArray(index, disableCols)) { + $input.attr('readonly', '').css('filter', 'invert(25%)'); + } + $cell.empty().append($input); + if (cell === current) $input.focus(); + $input.css('width', '100%'); + + if (immediate) $input.on('blur', function(e) { + var valueNew = $input.val(); + if (valueNew !== value) { + _cell.data(valueNew); + if (HTMLWidgets.shinyMode) { + changeInput('cell_edit', [cellInfo(cell)], 'DT.cellInfo', null, {priority: 'event'}); + } + // for server-side processing, users have to call replaceData() to update the table + if (!server) table.draw(false); + } else { + $cell.html(html); + } + }).on('keyup', function(e) { + // hit Escape to cancel editing + if (e.keyCode === 27) $input.trigger('blur'); + }); + + // bulk edit (row, column, or all) + if (!immediate) $input.on('keyup', function(e) { + var removeInput = function($cell, restore) { + $cell.data('input').remove(); + if (restore) $cell.html($cell.data('html')); + } + if (e.keyCode === 27) { + for (var i = 0; i < target.length; i++) { + removeInput($(target[i]), true); + } + } else if (e.keyCode === 13 && e.ctrlKey) { + // Ctrl + Enter + var cell, $cell, _cell, cellData = []; + for (var i = 0; i < target.length; i++) { + cell = target[i]; $cell = $(cell); _cell = table.cell(cell); + _cell.data($cell.data('input').val()); + HTMLWidgets.shinyMode && cellData.push(cellInfo(cell)); + removeInput($cell, false); + } + if (HTMLWidgets.shinyMode) { + changeInput('cell_edit', cellData, 'DT.cellInfo', null, {priority: "event"}); + } + if (!server) table.draw(false); + } + }); + })(target[i], this); + } + }); + + // interaction with shiny + if (!HTMLWidgets.shinyMode && !crosstalkOptions.group) return; + + var methods = {}; + var shinyData = {}; + + methods.updateCaption = function(caption) { + if (!caption) return; + $table.children('caption').replaceWith(caption); + } + + // register clear functions to remove input values when the table is removed + instance.clearInputs = {}; + + var changeInput = function(id, value, type, noCrosstalk, opts) { + var event = id; + id = el.id + '_' + id; + if (type) id = id + ':' + type; + // do not update if the new value is the same as old value + if (event !== 'cell_edit' && !/_clicked$/.test(event) && shinyData.hasOwnProperty(id) && shinyData[id] === JSON.stringify(value)) + return; + shinyData[id] = JSON.stringify(value); + if (HTMLWidgets.shinyMode && Shiny.setInputValue) { + Shiny.setInputValue(id, value, opts); + if (!instance.clearInputs[id]) instance.clearInputs[id] = function() { + Shiny.setInputValue(id, null); + } + } + + // HACK + if (event === "rows_selected" && !noCrosstalk) { + if (crosstalkOptions.group) { + var keys = crosstalkOptions.key; + var selectedKeys = null; + if (value) { + selectedKeys = []; + for (var i = 0; i < value.length; i++) { + // The value array's contents use 1-based row numbers, so we must + // convert to 0-based before indexing into the keys array. + selectedKeys.push(keys[value[i] - 1]); + } + } + instance.ctselectHandle.set(selectedKeys); + } + } + }; + + var addOne = function(x) { + return x.map(function(i) { return 1 + i; }); + }; + + var unique = function(x) { + var ux = []; + $.each(x, function(i, el){ + if ($.inArray(el, ux) === -1) ux.push(el); + }); + return ux; + } + + // change the row index of a cell + var tweakCellIndex = function(cell) { + var info = cell.index(); + // some cell may not be valid. e.g, #759 + // when using the RowGroup extension, datatables will + // generate the row label and the cells are not part of + // the data thus contain no row/col info + if (info === undefined) + return {row: null, col: null}; + if (server) { + info.row = DT_rows_current[info.row]; + } else { + info.row += 1; + } + return {row: info.row, col: info.column}; + } + + var cleanSelectedValues = function() { + changeInput('rows_selected', []); + changeInput('columns_selected', []); + changeInput('cells_selected', transposeArray2D([]), 'shiny.matrix'); + } + // #828 we should clean the selection on the server-side when the table reloads + cleanSelectedValues(); + + // a flag to indicates if select extension is initialized or not + var flagSelectExt = table.settings()[0]._select !== undefined; + // the Select extension should only be used in the client mode and + // when the selection.mode is set to none + if (data.selection.mode === 'none' && !server && flagSelectExt) { + var updateRowsSelected = function() { + var rows = table.rows({selected: true}); + var selected = []; + $.each(rows.indexes().toArray(), function(i, v) { + selected.push(v + 1); + }); + changeInput('rows_selected', selected); + } + var updateColsSelected = function() { + var columns = table.columns({selected: true}); + changeInput('columns_selected', columns.indexes().toArray()); + } + var updateCellsSelected = function() { + var cells = table.cells({selected: true}); + var selected = []; + cells.every(function() { + var row = this.index().row; + var col = this.index().column; + selected = selected.concat([[row + 1, col]]); + }); + changeInput('cells_selected', transposeArray2D(selected), 'shiny.matrix'); + } + table.on('select deselect', function(e, dt, type, indexes) { + updateRowsSelected(); + updateColsSelected(); + updateCellsSelected(); + }) + updateRowsSelected(); + updateColsSelected(); + updateCellsSelected(); + } + + var selMode = data.selection.mode, selTarget = data.selection.target; + var selDisable = data.selection.selectable === false; + if (inArray(selMode, ['single', 'multiple'])) { + var selClass = inArray(data.style, ['bootstrap', 'bootstrap4']) ? 'active' : 'selected'; + // selected1: row indices; selected2: column indices + var initSel = function(x) { + if (x === null || typeof x === 'boolean' || selTarget === 'cell') { + return {rows: [], cols: []}; + } else if (selTarget === 'row') { + return {rows: $.makeArray(x), cols: []}; + } else if (selTarget === 'column') { + return {rows: [], cols: $.makeArray(x)}; + } else if (selTarget === 'row+column') { + return {rows: $.makeArray(x.rows), cols: $.makeArray(x.cols)}; + } + } + var selected = data.selection.selected; + var selected1 = initSel(selected).rows, selected2 = initSel(selected).cols; + // selectable should contain either all positive or all non-positive values, not both + // positive values indicate "selectable" while non-positive values means "nonselectable" + // the assertion is performed on R side. (only column indicides could be zero which indicates + // the row name) + var selectable = data.selection.selectable; + var selectable1 = initSel(selectable).rows, selectable2 = initSel(selectable).cols; + + // After users reorder the rows or filter the table, we cannot use the table index + // directly. Instead, we need this function to find out the rows between the two clicks. + // If user filter the table again between the start click and the end click, the behavior + // would be undefined, but it should not be a problem. + var shiftSelRowsIndex = function(start, end) { + var indexes = server ? DT_rows_all : table.rows({ search: 'applied' }).indexes().toArray(); + start = indexes.indexOf(start); end = indexes.indexOf(end); + // if start is larger than end, we need to swap + if (start > end) { + var tmp = end; end = start; start = tmp; + } + return indexes.slice(start, end + 1); + } + + var serverRowIndex = function(clientRowIndex) { + return server ? DT_rows_current[clientRowIndex] : clientRowIndex + 1; + } + + // row, column, or cell selection + var lastClickedRow; + if (inArray(selTarget, ['row', 'row+column'])) { + // Get the current selected rows. It will also + // update the selected1's value based on the current row selection state + // Note we can't put this function inside selectRows() directly, + // the reason is method.selectRows() will override selected1's value but this + // function will add rows to selected1 (keep the existing selection), which is + // inconsistent with column and cell selection. + var selectedRows = function() { + var rows = table.rows('.' + selClass); + var idx = rows.indexes().toArray(); + if (!server) { + selected1 = addOne(idx); + return selected1; + } + idx = idx.map(function(i) { + return DT_rows_current[i]; + }); + selected1 = selMode === 'multiple' ? unique(selected1.concat(idx)) : idx; + return selected1; + } + // Change selected1's value based on selectable1, then refresh the row state + var onlyKeepSelectableRows = function() { + if (selDisable) { // users can't select; useful when only want backend select + selected1 = []; + return; + } + if (selectable1.length === 0) return; + var nonselectable = selectable1[0] <= 0; + if (nonselectable) { + // should make selectable1 positive + selected1 = $(selected1).not(selectable1.map(function(i) { return -i; })).get(); + } else { + selected1 = $(selected1).filter(selectable1).get(); + } + } + // Change selected1's value based on selectable1, then + // refresh the row selection state according to values in selected1 + var selectRows = function(ignoreSelectable) { + if (!ignoreSelectable) onlyKeepSelectableRows(); + table.$('tr.' + selClass).removeClass(selClass); + if (selected1.length === 0) return; + if (server) { + table.rows({page: 'current'}).every(function() { + if (inArray(DT_rows_current[this.index()], selected1)) { + $(this.node()).addClass(selClass); + } + }); + } else { + var selected0 = selected1.map(function(i) { return i - 1; }); + $(table.rows(selected0).nodes()).addClass(selClass); + } + } + table.on('mousedown.dt', 'tbody tr', function(e) { + var $this = $(this), thisRow = table.row(this); + if (selMode === 'multiple') { + if (e.shiftKey && lastClickedRow !== undefined) { + // select or de-select depends on the last clicked row's status + var flagSel = !$this.hasClass(selClass); + var crtClickedRow = serverRowIndex(thisRow.index()); + if (server) { + var rowsIndex = shiftSelRowsIndex(lastClickedRow, crtClickedRow); + // update current page's selClass + rowsIndex.map(function(i) { + var rowIndex = DT_rows_current.indexOf(i); + if (rowIndex >= 0) { + var row = table.row(rowIndex).nodes().to$(); + var flagRowSel = !row.hasClass(selClass); + if (flagSel === flagRowSel) row.toggleClass(selClass); + } + }); + // update selected1 + if (flagSel) { + selected1 = unique(selected1.concat(rowsIndex)); + } else { + selected1 = selected1.filter(function(index) { + return !inArray(index, rowsIndex); + }); + } + } else { + // js starts from 0 + shiftSelRowsIndex(lastClickedRow - 1, crtClickedRow - 1).map(function(value) { + var row = table.row(value).nodes().to$(); + var flagRowSel = !row.hasClass(selClass); + if (flagSel === flagRowSel) row.toggleClass(selClass); + }); + } + e.preventDefault(); + } else { + $this.toggleClass(selClass); + } + } else { + if ($this.hasClass(selClass)) { + $this.removeClass(selClass); + } else { + table.$('tr.' + selClass).removeClass(selClass); + $this.addClass(selClass); + } + } + if (server && !$this.hasClass(selClass)) { + var id = DT_rows_current[thisRow.index()]; + // remove id from selected1 since its class .selected has been removed + if (inArray(id, selected1)) selected1.splice($.inArray(id, selected1), 1); + } + selectedRows(); // update selected1's value based on selClass + selectRows(false); // only keep the selectable rows + changeInput('rows_selected', selected1); + changeInput('row_last_clicked', serverRowIndex(thisRow.index()), null, null, {priority: 'event'}); + lastClickedRow = serverRowIndex(thisRow.index()); + }); + selectRows(false); // in case users have specified pre-selected rows + // restore selected rows after the table is redrawn (e.g. sort/search/page); + // client-side tables will preserve the selections automatically; for + // server-side tables, we have to *real* row indices are in `selected1` + changeInput('rows_selected', selected1); + if (server) table.on('draw.dt', function(e) { selectRows(false); }); + methods.selectRows = function(selected, ignoreSelectable) { + selected1 = $.makeArray(selected); + selectRows(ignoreSelectable); + changeInput('rows_selected', selected1); + } + } + + if (inArray(selTarget, ['column', 'row+column'])) { + if (selTarget === 'row+column') { + $(table.columns().footer()).css('cursor', 'pointer'); + } + // update selected2's value based on selectable2 + var onlyKeepSelectableCols = function() { + if (selDisable) { // users can't select; useful when only want backend select + selected2 = []; + return; + } + if (selectable2.length === 0) return; + var nonselectable = selectable2[0] <= 0; + if (nonselectable) { + // need to make selectable2 positive + selected2 = $(selected2).not(selectable2.map(function(i) { return -i; })).get(); + } else { + selected2 = $(selected2).filter(selectable2).get(); + } + } + // update selected2 and then + // refresh the col selection state according to values in selected2 + var selectCols = function(ignoreSelectable) { + if (!ignoreSelectable) onlyKeepSelectableCols(); + // if selected2 is not a valide index (e.g., larger than the column number) + // table.columns(selected2) will fail and result in a blank table + // this is different from the table.rows(), where the out-of-range indexes + // doesn't affect at all + selected2 = $(selected2).filter(table.columns().indexes()).get(); + table.columns().nodes().flatten().to$().removeClass(selClass); + if (selected2.length > 0) + table.columns(selected2).nodes().flatten().to$().addClass(selClass); + } + var callback = function() { + var colIdx = selTarget === 'column' ? table.cell(this).index().column : + $.inArray(this, table.columns().footer()), + thisCol = $(table.column(colIdx).nodes()); + if (colIdx === -1) return; + if (thisCol.hasClass(selClass)) { + thisCol.removeClass(selClass); + selected2.splice($.inArray(colIdx, selected2), 1); + } else { + if (selMode === 'single') $(table.cells().nodes()).removeClass(selClass); + thisCol.addClass(selClass); + selected2 = selMode === 'single' ? [colIdx] : unique(selected2.concat([colIdx])); + } + selectCols(false); // update selected2 based on selectable + changeInput('columns_selected', selected2); + } + if (selTarget === 'column') { + $(table.table().body()).on('click.dt', 'td', callback); + } else { + $(table.table().footer()).on('click.dt', 'tr th', callback); + } + selectCols(false); // in case users have specified pre-selected columns + changeInput('columns_selected', selected2); + if (server) table.on('draw.dt', function(e) { selectCols(false); }); + methods.selectColumns = function(selected, ignoreSelectable) { + selected2 = $.makeArray(selected); + selectCols(ignoreSelectable); + changeInput('columns_selected', selected2); + } + } + + if (selTarget === 'cell') { + var selected3 = [], selectable3 = []; + if (selected !== null) selected3 = selected; + if (selectable !== null && typeof selectable !== 'boolean') selectable3 = selectable; + var findIndex = function(ij, sel) { + for (var i = 0; i < sel.length; i++) { + if (ij[0] === sel[i][0] && ij[1] === sel[i][1]) return i; + } + return -1; + } + // Change selected3's value based on selectable3, then refresh the cell state + var onlyKeepSelectableCells = function() { + if (selDisable) { // users can't select; useful when only want backend select + selected3 = []; + return; + } + if (selectable3.length === 0) return; + var nonselectable = selectable3[0][0] <= 0; + var out = []; + if (nonselectable) { + selected3.map(function(ij) { + // should make selectable3 positive + if (findIndex([-ij[0], -ij[1]], selectable3) === -1) { out.push(ij); } + }); + } else { + selected3.map(function(ij) { + if (findIndex(ij, selectable3) > -1) { out.push(ij); } + }); + } + selected3 = out; + } + // Change selected3's value based on selectable3, then + // refresh the cell selection state according to values in selected3 + var selectCells = function(ignoreSelectable) { + if (!ignoreSelectable) onlyKeepSelectableCells(); + table.$('td.' + selClass).removeClass(selClass); + if (selected3.length === 0) return; + if (server) { + table.cells({page: 'current'}).every(function() { + var info = tweakCellIndex(this); + if (findIndex([info.row, info.col], selected3) > -1) + $(this.node()).addClass(selClass); + }); + } else { + selected3.map(function(ij) { + $(table.cell(ij[0] - 1, ij[1]).node()).addClass(selClass); + }); + } + }; + table.on('click.dt', 'tbody td', function() { + var $this = $(this), info = tweakCellIndex(table.cell(this)); + if ($this.hasClass(selClass)) { + $this.removeClass(selClass); + selected3.splice(findIndex([info.row, info.col], selected3), 1); + } else { + if (selMode === 'single') $(table.cells().nodes()).removeClass(selClass); + $this.addClass(selClass); + selected3 = selMode === 'single' ? [[info.row, info.col]] : + unique(selected3.concat([[info.row, info.col]])); + } + selectCells(false); // must call this to update selected3 based on selectable3 + changeInput('cells_selected', transposeArray2D(selected3), 'shiny.matrix'); + }); + selectCells(false); // in case users have specified pre-selected columns + changeInput('cells_selected', transposeArray2D(selected3), 'shiny.matrix'); + + if (server) table.on('draw.dt', function(e) { selectCells(false); }); + methods.selectCells = function(selected, ignoreSelectable) { + selected3 = selected ? selected : []; + selectCells(ignoreSelectable); + changeInput('cells_selected', transposeArray2D(selected3), 'shiny.matrix'); + } + } + } + + // expose some table info to Shiny + var updateTableInfo = function(e, settings) { + // TODO: is anyone interested in the page info? + // changeInput('page_info', table.page.info()); + var updateRowInfo = function(id, modifier) { + var idx; + if (server) { + idx = modifier.page === 'current' ? DT_rows_current : DT_rows_all; + } else { + var rows = table.rows($.extend({ + search: 'applied', + page: 'all' + }, modifier)); + idx = addOne(rows.indexes().toArray()); + } + changeInput('rows' + '_' + id, idx); + }; + updateRowInfo('current', {page: 'current'}); + updateRowInfo('all', {}); + } + table.on('draw.dt', updateTableInfo); + updateTableInfo(); + + // state info + table.on('draw.dt column-visibility.dt', function() { + changeInput('state', table.state()); + }); + changeInput('state', table.state()); + + // search info + var updateSearchInfo = function() { + changeInput('search', table.search()); + if (filterRow) changeInput('search_columns', filterRow.toArray().map(function(td) { + return $(td).find('input').first().val(); + })); + } + table.on('draw.dt', updateSearchInfo); + updateSearchInfo(); + + var cellInfo = function(thiz) { + var info = tweakCellIndex(table.cell(thiz)); + info.value = table.cell(thiz).data(); + return info; + } + // the current cell clicked on + table.on('click.dt', 'tbody td', function() { + changeInput('cell_clicked', cellInfo(this), null, null, {priority: 'event'}); + }) + changeInput('cell_clicked', {}); + + // do not trigger table selection when clicking on links unless they have classes + table.on('mousedown.dt', 'tbody td a', function(e) { + if (this.className === '') e.stopPropagation(); + }); + + methods.addRow = function(data, rowname, resetPaging) { + var n = table.columns().indexes().length, d = n - data.length; + if (d === 1) { + data = rowname.concat(data) + } else if (d !== 0) { + console.log(data); + console.log(table.columns().indexes()); + throw 'New data must be of the same length as current data (' + n + ')'; + }; + table.row.add(data).draw(resetPaging); + } + + methods.updateSearch = function(keywords) { + if (keywords.global !== null) + $(table.table().container()).find('input[type=search]').first() + .val(keywords.global).trigger('input'); + var columns = keywords.columns; + if (!filterRow || columns === null) return; + filterRow.toArray().map(function(td, i) { + var v = typeof columns === 'string' ? columns : columns[i]; + if (typeof v === 'undefined') { + console.log('The search keyword for column ' + i + ' is undefined') + return; + } + // Update column search string and values on linked filter widgets. + // 'input' for factor and char filters, 'change' for numeric filters. + $(td).find('input').first().val(v).trigger('input', [true]).trigger('change'); + }); + table.draw(); + } + + methods.hideCols = function(hide, reset) { + if (reset) table.columns().visible(true, false); + table.columns(hide).visible(false); + } + + methods.showCols = function(show, reset) { + if (reset) table.columns().visible(false, false); + table.columns(show).visible(true); + } + + methods.colReorder = function(order, origOrder) { + table.colReorder.order(order, origOrder); + } + + methods.selectPage = function(page) { + if (table.page.info().pages < page || page < 1) { + throw 'Selected page is out of range'; + }; + table.page(page - 1).draw(false); + } + + methods.reloadData = function(resetPaging, clearSelection) { + // empty selections first if necessary + if (methods.selectRows && inArray('row', clearSelection)) methods.selectRows([]); + if (methods.selectColumns && inArray('column', clearSelection)) methods.selectColumns([]); + if (methods.selectCells && inArray('cell', clearSelection)) methods.selectCells([]); + table.ajax.reload(null, resetPaging); + } + + // update table filters (set new limits of sliders) + methods.updateFilters = function(newProps) { + // loop through each filter in the filter row + filterRow.each(function(i, td) { + var k = i; + if (filterRow.length > newProps.length) { + if (i === 0) return; // first column is row names + k = i - 1; + } + // Update the filters to reflect the updated data. + // Allow "falsy" (e.g. NULL) to signify a no-op. + if (newProps[k]) { + setFilterProps(td, newProps[k]); + } + }); + }; + + table.shinyMethods = methods; + }, + resize: function(el, width, height, instance) { + if (instance.data) this.renderValue(el, instance.data, instance); + + // dynamically adjust height if fillContainer = TRUE + if (instance.fillContainer) + this.fillAvailableHeight(el, height); + + this.adjustWidth(el); + }, + + // dynamically set the scroll body to fill available height + // (used with fillContainer = TRUE) + fillAvailableHeight: function(el, availableHeight) { + + // see how much of the table is occupied by header/footer elements + // and use that to compute a target scroll body height + var dtWrapper = $(el).find('div.dataTables_wrapper'); + var dtScrollBody = $(el).find($('div.dataTables_scrollBody')); + var framingHeight = dtWrapper.innerHeight() - dtScrollBody.innerHeight(); + var scrollBodyHeight = availableHeight - framingHeight; + + // we need to set `max-height` to none as datatables library now sets this + // to a fixed height, disabling the ability to resize to fill the window, + // as it will be set to a fixed 100px under such circumstances, e.g., RStudio IDE, + // or FlexDashboard + // see https://github.com/rstudio/DT/issues/951#issuecomment-1026464509 + dtScrollBody.css('max-height', 'none'); + // set the height + dtScrollBody.height(scrollBodyHeight + 'px'); + }, + + // adjust the width of columns; remove the hard-coded widths on table and the + // scroll header when scrollX/Y are enabled + adjustWidth: function(el) { + var $el = $(el), table = $el.data('datatable'); + if (table) table.columns.adjust(); + $el.find('.dataTables_scrollHeadInner').css('width', '') + .children('table').css('margin-left', ''); + } +}); + + if (!HTMLWidgets.shinyMode) return; + + Shiny.addCustomMessageHandler('datatable-calls', function(data) { + var id = data.id; + var el = document.getElementById(id); + var table = el ? $(el).data('datatable') : null; + if (!table) { + console.log("Couldn't find table with id " + id); + return; + } + + var methods = table.shinyMethods, call = data.call; + if (methods[call.method]) { + methods[call.method].apply(table, call.args); + } else { + console.log("Unknown method " + call.method); + } + }); + +})(); diff --git a/docs/site_libs/datatables-css-0.0.0/datatables-crosstalk.css b/docs/site_libs/datatables-css-0.0.0/datatables-crosstalk.css new file mode 100644 index 00000000..bd1159c8 --- /dev/null +++ b/docs/site_libs/datatables-css-0.0.0/datatables-crosstalk.css @@ -0,0 +1,32 @@ +.dt-crosstalk-fade { + opacity: 0.2; +} + +html body div.DTS div.dataTables_scrollBody { + background: none; +} + + +/* +Fix https://github.com/rstudio/DT/issues/563 +If the `table.display` is set to "block" (e.g., pkgdown), the browser will display +datatable objects strangely. The search panel and the page buttons will still be +in full-width but the table body will be "compact" and shorter. +In therory, having this attributes will affect `dom="t"` +with `display: block` users. But in reality, there should be no one. +We may remove the below lines in the future if the upstream agree to have this there. +See https://github.com/DataTables/DataTablesSrc/issues/160 +*/ + +table.dataTable { + display: table; +} + + +/* +When DTOutput(fill = TRUE), it receives a .html-fill-item class (via htmltools::bindFillRole()), which effectively amounts to `flex: 1 1 auto`. That's mostly fine, but the case where `fillContainer=TRUE`+`height:auto`+`flex-basis:auto` and the container (e.g., a bslib::card()) doesn't have a defined height is a bit problematic since the table wants to fit the parent but the parent wants to fit the table, which results pretty small table height (maybe because there is a minimum height somewhere?). It seems better in this case to impose a 400px height default for the table, which we can do by setting `flex-basis` to 400px (the table is still allowed to grow/shrink when the container has an opinionated height). +*/ + +.html-fill-container > .html-fill-item.datatables { + flex-basis: 400px; +} diff --git a/docs/site_libs/dt-core-1.13.6/css/jquery.dataTables.extra.css b/docs/site_libs/dt-core-1.13.6/css/jquery.dataTables.extra.css new file mode 100644 index 00000000..b2dd141f --- /dev/null +++ b/docs/site_libs/dt-core-1.13.6/css/jquery.dataTables.extra.css @@ -0,0 +1,28 @@ +/* Selected rows/cells */ +table.dataTable tr.selected td, table.dataTable td.selected { + background-color: #b0bed9 !important; +} +/* In case of scrollX/Y or FixedHeader */ +.dataTables_scrollBody .dataTables_sizing { + visibility: hidden; +} + +/* The datatables' theme CSS file doesn't define +the color but with white background. It leads to an issue that +when the HTML's body color is set to 'white', the user can't +see the text since the background is white. One case happens in the +RStudio's IDE when inline viewing the DT table inside an Rmd file, +if the IDE theme is set to "Cobalt". + +See https://github.com/rstudio/DT/issues/447 for more info + +This fixes should have little side-effects because all the other elements +of the default theme use the #333 font color. + +TODO: The upstream may use relative colors for both the table background +and the color. It means the table can display well without this patch +then. At that time, we need to remove the below CSS attributes. +*/ +div.datatables { + color: #333; +} diff --git a/docs/site_libs/dt-core-1.13.6/css/jquery.dataTables.min.css b/docs/site_libs/dt-core-1.13.6/css/jquery.dataTables.min.css new file mode 100644 index 00000000..ad59f843 --- /dev/null +++ b/docs/site_libs/dt-core-1.13.6/css/jquery.dataTables.min.css @@ -0,0 +1 @@ +:root{--dt-row-selected: 13, 110, 253;--dt-row-selected-text: 255, 255, 255;--dt-row-selected-link: 9, 10, 11;--dt-row-stripe: 0, 0, 0;--dt-row-hover: 0, 0, 0;--dt-column-ordering: 0, 0, 0;--dt-html-background: white}:root.dark{--dt-html-background: rgb(33, 37, 41)}table.dataTable td.dt-control{text-align:center;cursor:pointer}table.dataTable td.dt-control:before{display:inline-block;color:rgba(0, 0, 0, 0.5);content:"►"}table.dataTable tr.dt-hasChild td.dt-control:before{content:"▼"}html.dark table.dataTable td.dt-control:before{color:rgba(255, 255, 255, 0.5)}html.dark table.dataTable tr.dt-hasChild td.dt-control:before{color:rgba(255, 255, 255, 0.5)}table.dataTable thead>tr>th.sorting,table.dataTable thead>tr>th.sorting_asc,table.dataTable thead>tr>th.sorting_desc,table.dataTable thead>tr>th.sorting_asc_disabled,table.dataTable thead>tr>th.sorting_desc_disabled,table.dataTable thead>tr>td.sorting,table.dataTable thead>tr>td.sorting_asc,table.dataTable thead>tr>td.sorting_desc,table.dataTable thead>tr>td.sorting_asc_disabled,table.dataTable thead>tr>td.sorting_desc_disabled{cursor:pointer;position:relative;padding-right:26px}table.dataTable thead>tr>th.sorting:before,table.dataTable thead>tr>th.sorting:after,table.dataTable thead>tr>th.sorting_asc:before,table.dataTable thead>tr>th.sorting_asc:after,table.dataTable thead>tr>th.sorting_desc:before,table.dataTable thead>tr>th.sorting_desc:after,table.dataTable thead>tr>th.sorting_asc_disabled:before,table.dataTable thead>tr>th.sorting_asc_disabled:after,table.dataTable thead>tr>th.sorting_desc_disabled:before,table.dataTable thead>tr>th.sorting_desc_disabled:after,table.dataTable thead>tr>td.sorting:before,table.dataTable thead>tr>td.sorting:after,table.dataTable thead>tr>td.sorting_asc:before,table.dataTable thead>tr>td.sorting_asc:after,table.dataTable thead>tr>td.sorting_desc:before,table.dataTable thead>tr>td.sorting_desc:after,table.dataTable thead>tr>td.sorting_asc_disabled:before,table.dataTable thead>tr>td.sorting_asc_disabled:after,table.dataTable thead>tr>td.sorting_desc_disabled:before,table.dataTable thead>tr>td.sorting_desc_disabled:after{position:absolute;display:block;opacity:.125;right:10px;line-height:9px;font-size:.8em}table.dataTable thead>tr>th.sorting:before,table.dataTable thead>tr>th.sorting_asc:before,table.dataTable thead>tr>th.sorting_desc:before,table.dataTable thead>tr>th.sorting_asc_disabled:before,table.dataTable thead>tr>th.sorting_desc_disabled:before,table.dataTable thead>tr>td.sorting:before,table.dataTable thead>tr>td.sorting_asc:before,table.dataTable thead>tr>td.sorting_desc:before,table.dataTable thead>tr>td.sorting_asc_disabled:before,table.dataTable thead>tr>td.sorting_desc_disabled:before{bottom:50%;content:"▲";content:"▲"/""}table.dataTable thead>tr>th.sorting:after,table.dataTable thead>tr>th.sorting_asc:after,table.dataTable thead>tr>th.sorting_desc:after,table.dataTable thead>tr>th.sorting_asc_disabled:after,table.dataTable thead>tr>th.sorting_desc_disabled:after,table.dataTable thead>tr>td.sorting:after,table.dataTable thead>tr>td.sorting_asc:after,table.dataTable thead>tr>td.sorting_desc:after,table.dataTable thead>tr>td.sorting_asc_disabled:after,table.dataTable thead>tr>td.sorting_desc_disabled:after{top:50%;content:"▼";content:"▼"/""}table.dataTable thead>tr>th.sorting_asc:before,table.dataTable thead>tr>th.sorting_desc:after,table.dataTable thead>tr>td.sorting_asc:before,table.dataTable thead>tr>td.sorting_desc:after{opacity:.6}table.dataTable thead>tr>th.sorting_desc_disabled:after,table.dataTable thead>tr>th.sorting_asc_disabled:before,table.dataTable thead>tr>td.sorting_desc_disabled:after,table.dataTable thead>tr>td.sorting_asc_disabled:before{display:none}table.dataTable thead>tr>th:active,table.dataTable thead>tr>td:active{outline:none}div.dataTables_scrollBody>table.dataTable>thead>tr>th:before,div.dataTables_scrollBody>table.dataTable>thead>tr>th:after,div.dataTables_scrollBody>table.dataTable>thead>tr>td:before,div.dataTables_scrollBody>table.dataTable>thead>tr>td:after{display:none}div.dataTables_processing{position:absolute;top:50%;left:50%;width:200px;margin-left:-100px;margin-top:-26px;text-align:center;padding:2px}div.dataTables_processing>div:last-child{position:relative;width:80px;height:15px;margin:1em auto}div.dataTables_processing>div:last-child>div{position:absolute;top:0;width:13px;height:13px;border-radius:50%;background:rgb(13, 110, 253);background:rgb(var(--dt-row-selected));animation-timing-function:cubic-bezier(0, 1, 1, 0)}div.dataTables_processing>div:last-child>div:nth-child(1){left:8px;animation:datatables-loader-1 .6s infinite}div.dataTables_processing>div:last-child>div:nth-child(2){left:8px;animation:datatables-loader-2 .6s infinite}div.dataTables_processing>div:last-child>div:nth-child(3){left:32px;animation:datatables-loader-2 .6s infinite}div.dataTables_processing>div:last-child>div:nth-child(4){left:56px;animation:datatables-loader-3 .6s infinite}@keyframes datatables-loader-1{0%{transform:scale(0)}100%{transform:scale(1)}}@keyframes datatables-loader-3{0%{transform:scale(1)}100%{transform:scale(0)}}@keyframes datatables-loader-2{0%{transform:translate(0, 0)}100%{transform:translate(24px, 0)}}table.dataTable.nowrap th,table.dataTable.nowrap td{white-space:nowrap}table.dataTable th.dt-left,table.dataTable td.dt-left{text-align:left}table.dataTable th.dt-center,table.dataTable td.dt-center,table.dataTable td.dataTables_empty{text-align:center}table.dataTable th.dt-right,table.dataTable td.dt-right{text-align:right}table.dataTable th.dt-justify,table.dataTable td.dt-justify{text-align:justify}table.dataTable th.dt-nowrap,table.dataTable td.dt-nowrap{white-space:nowrap}table.dataTable thead th,table.dataTable thead td,table.dataTable tfoot th,table.dataTable tfoot td{text-align:left}table.dataTable thead th.dt-head-left,table.dataTable thead td.dt-head-left,table.dataTable tfoot th.dt-head-left,table.dataTable tfoot td.dt-head-left{text-align:left}table.dataTable thead th.dt-head-center,table.dataTable thead td.dt-head-center,table.dataTable tfoot th.dt-head-center,table.dataTable tfoot td.dt-head-center{text-align:center}table.dataTable thead th.dt-head-right,table.dataTable thead td.dt-head-right,table.dataTable tfoot th.dt-head-right,table.dataTable tfoot td.dt-head-right{text-align:right}table.dataTable thead th.dt-head-justify,table.dataTable thead td.dt-head-justify,table.dataTable tfoot th.dt-head-justify,table.dataTable tfoot td.dt-head-justify{text-align:justify}table.dataTable thead th.dt-head-nowrap,table.dataTable thead td.dt-head-nowrap,table.dataTable tfoot th.dt-head-nowrap,table.dataTable tfoot td.dt-head-nowrap{white-space:nowrap}table.dataTable tbody th.dt-body-left,table.dataTable tbody td.dt-body-left{text-align:left}table.dataTable tbody th.dt-body-center,table.dataTable tbody td.dt-body-center{text-align:center}table.dataTable tbody th.dt-body-right,table.dataTable tbody td.dt-body-right{text-align:right}table.dataTable tbody th.dt-body-justify,table.dataTable tbody td.dt-body-justify{text-align:justify}table.dataTable tbody th.dt-body-nowrap,table.dataTable tbody td.dt-body-nowrap{white-space:nowrap}table.dataTable{width:100%;margin:0 auto;clear:both;border-collapse:separate;border-spacing:0}table.dataTable thead th,table.dataTable tfoot th{font-weight:bold}table.dataTable>thead>tr>th,table.dataTable>thead>tr>td{padding:10px;border-bottom:1px solid rgba(0, 0, 0, 0.3)}table.dataTable>thead>tr>th:active,table.dataTable>thead>tr>td:active{outline:none}table.dataTable>tfoot>tr>th,table.dataTable>tfoot>tr>td{padding:10px 10px 6px 10px;border-top:1px solid rgba(0, 0, 0, 0.3)}table.dataTable tbody tr{background-color:transparent}table.dataTable tbody tr.selected>*{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.9);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.9);color:rgb(255, 255, 255);color:rgb(var(--dt-row-selected-text))}table.dataTable tbody tr.selected a{color:rgb(9, 10, 11);color:rgb(var(--dt-row-selected-link))}table.dataTable tbody th,table.dataTable tbody td{padding:8px 10px}table.dataTable.row-border>tbody>tr>th,table.dataTable.row-border>tbody>tr>td,table.dataTable.display>tbody>tr>th,table.dataTable.display>tbody>tr>td{border-top:1px solid rgba(0, 0, 0, 0.15)}table.dataTable.row-border>tbody>tr:first-child>th,table.dataTable.row-border>tbody>tr:first-child>td,table.dataTable.display>tbody>tr:first-child>th,table.dataTable.display>tbody>tr:first-child>td{border-top:none}table.dataTable.row-border>tbody>tr.selected+tr.selected>td,table.dataTable.display>tbody>tr.selected+tr.selected>td{border-top-color:#0262ef}table.dataTable.cell-border>tbody>tr>th,table.dataTable.cell-border>tbody>tr>td{border-top:1px solid rgba(0, 0, 0, 0.15);border-right:1px solid rgba(0, 0, 0, 0.15)}table.dataTable.cell-border>tbody>tr>th:first-child,table.dataTable.cell-border>tbody>tr>td:first-child{border-left:1px solid rgba(0, 0, 0, 0.15)}table.dataTable.cell-border>tbody>tr:first-child>th,table.dataTable.cell-border>tbody>tr:first-child>td{border-top:none}table.dataTable.stripe>tbody>tr.odd>*,table.dataTable.display>tbody>tr.odd>*{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.023);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-stripe), 0.023)}table.dataTable.stripe>tbody>tr.odd.selected>*,table.dataTable.display>tbody>tr.odd.selected>*{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.923);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.923)}table.dataTable.hover>tbody>tr:hover>*,table.dataTable.display>tbody>tr:hover>*{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.035);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.035)}table.dataTable.hover>tbody>tr.selected:hover>*,table.dataTable.display>tbody>tr.selected:hover>*{box-shadow:inset 0 0 0 9999px #0d6efd !important;box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 1) !important}table.dataTable.order-column>tbody tr>.sorting_1,table.dataTable.order-column>tbody tr>.sorting_2,table.dataTable.order-column>tbody tr>.sorting_3,table.dataTable.display>tbody tr>.sorting_1,table.dataTable.display>tbody tr>.sorting_2,table.dataTable.display>tbody tr>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.019);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.019)}table.dataTable.order-column>tbody tr.selected>.sorting_1,table.dataTable.order-column>tbody tr.selected>.sorting_2,table.dataTable.order-column>tbody tr.selected>.sorting_3,table.dataTable.display>tbody tr.selected>.sorting_1,table.dataTable.display>tbody tr.selected>.sorting_2,table.dataTable.display>tbody tr.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.919);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.919)}table.dataTable.display>tbody>tr.odd>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.odd>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.054);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.054)}table.dataTable.display>tbody>tr.odd>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.odd>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.047);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.047)}table.dataTable.display>tbody>tr.odd>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.odd>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.039);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.039)}table.dataTable.display>tbody>tr.odd.selected>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.odd.selected>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.954);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.954)}table.dataTable.display>tbody>tr.odd.selected>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.odd.selected>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.947);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.947)}table.dataTable.display>tbody>tr.odd.selected>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.odd.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.939);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.939)}table.dataTable.display>tbody>tr.even>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.even>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.019);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.019)}table.dataTable.display>tbody>tr.even>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.even>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.011);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.011)}table.dataTable.display>tbody>tr.even>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.even>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.003);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.003)}table.dataTable.display>tbody>tr.even.selected>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.even.selected>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.919);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.919)}table.dataTable.display>tbody>tr.even.selected>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.even.selected>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.911);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.911)}table.dataTable.display>tbody>tr.even.selected>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.even.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.903);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.903)}table.dataTable.display tbody tr:hover>.sorting_1,table.dataTable.order-column.hover tbody tr:hover>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.082);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.082)}table.dataTable.display tbody tr:hover>.sorting_2,table.dataTable.order-column.hover tbody tr:hover>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.074);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.074)}table.dataTable.display tbody tr:hover>.sorting_3,table.dataTable.order-column.hover tbody tr:hover>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.062);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.062)}table.dataTable.display tbody tr:hover.selected>.sorting_1,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.982);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.982)}table.dataTable.display tbody tr:hover.selected>.sorting_2,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.974);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.974)}table.dataTable.display tbody tr:hover.selected>.sorting_3,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.962);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.962)}table.dataTable.no-footer{border-bottom:1px solid rgba(0, 0, 0, 0.3)}table.dataTable.compact thead th,table.dataTable.compact thead td,table.dataTable.compact tfoot th,table.dataTable.compact tfoot td,table.dataTable.compact tbody th,table.dataTable.compact tbody td{padding:4px}table.dataTable th,table.dataTable td{box-sizing:content-box}.dataTables_wrapper{position:relative;clear:both}.dataTables_wrapper .dataTables_length{float:left}.dataTables_wrapper .dataTables_length select{border:1px solid #aaa;border-radius:3px;padding:5px;background-color:transparent;color:inherit;padding:4px}.dataTables_wrapper .dataTables_filter{float:right;text-align:right}.dataTables_wrapper .dataTables_filter input{border:1px solid #aaa;border-radius:3px;padding:5px;background-color:transparent;color:inherit;margin-left:3px}.dataTables_wrapper .dataTables_info{clear:both;float:left;padding-top:.755em}.dataTables_wrapper .dataTables_paginate{float:right;text-align:right;padding-top:.25em}.dataTables_wrapper .dataTables_paginate .paginate_button{box-sizing:border-box;display:inline-block;min-width:1.5em;padding:.5em 1em;margin-left:2px;text-align:center;text-decoration:none !important;cursor:pointer;color:inherit !important;border:1px solid transparent;border-radius:2px;background:transparent}.dataTables_wrapper .dataTables_paginate .paginate_button.current,.dataTables_wrapper .dataTables_paginate .paginate_button.current:hover{color:inherit !important;border:1px solid rgba(0, 0, 0, 0.3);background-color:rgba(0, 0, 0, 0.05);background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, rgba(230, 230, 230, 0.05)), color-stop(100%, rgba(0, 0, 0, 0.05)));background:-webkit-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:-moz-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:-ms-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:-o-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:linear-gradient(to bottom, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button.disabled,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active{cursor:default;color:#666 !important;border:1px solid transparent;background:transparent;box-shadow:none}.dataTables_wrapper .dataTables_paginate .paginate_button:hover{color:white !important;border:1px solid #111;background-color:#111;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111));background:-webkit-linear-gradient(top, #585858 0%, #111 100%);background:-moz-linear-gradient(top, #585858 0%, #111 100%);background:-ms-linear-gradient(top, #585858 0%, #111 100%);background:-o-linear-gradient(top, #585858 0%, #111 100%);background:linear-gradient(to bottom, #585858 0%, #111 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button:active{outline:none;background-color:#0c0c0c;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c));background:-webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%);box-shadow:inset 0 0 3px #111}.dataTables_wrapper .dataTables_paginate .ellipsis{padding:0 1em}.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter,.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_processing,.dataTables_wrapper .dataTables_paginate{color:inherit}.dataTables_wrapper .dataTables_scroll{clear:both}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody{-webkit-overflow-scrolling:touch}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>th,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>td,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>th,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>td{vertical-align:middle}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>th>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>td>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>th>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>td>div.dataTables_sizing{height:0;overflow:hidden;margin:0 !important;padding:0 !important}.dataTables_wrapper.no-footer .dataTables_scrollBody{border-bottom:1px solid rgba(0, 0, 0, 0.3)}.dataTables_wrapper.no-footer div.dataTables_scrollHead table.dataTable,.dataTables_wrapper.no-footer div.dataTables_scrollBody>table{border-bottom:none}.dataTables_wrapper:after{visibility:hidden;display:block;content:"";clear:both;height:0}@media screen and (max-width: 767px){.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_paginate{float:none;text-align:center}.dataTables_wrapper .dataTables_paginate{margin-top:.5em}}@media screen and (max-width: 640px){.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter{float:none;text-align:center}.dataTables_wrapper .dataTables_filter{margin-top:.5em}}html.dark{--dt-row-hover: 255, 255, 255;--dt-row-stripe: 255, 255, 255;--dt-column-ordering: 255, 255, 255}html.dark table.dataTable>thead>tr>th,html.dark table.dataTable>thead>tr>td{border-bottom:1px solid rgb(89, 91, 94)}html.dark table.dataTable>thead>tr>th:active,html.dark table.dataTable>thead>tr>td:active{outline:none}html.dark table.dataTable>tfoot>tr>th,html.dark table.dataTable>tfoot>tr>td{border-top:1px solid rgb(89, 91, 94)}html.dark table.dataTable.row-border>tbody>tr>th,html.dark table.dataTable.row-border>tbody>tr>td,html.dark table.dataTable.display>tbody>tr>th,html.dark table.dataTable.display>tbody>tr>td{border-top:1px solid rgb(64, 67, 70)}html.dark table.dataTable.row-border>tbody>tr.selected+tr.selected>td,html.dark table.dataTable.display>tbody>tr.selected+tr.selected>td{border-top-color:#0257d5}html.dark table.dataTable.cell-border>tbody>tr>th,html.dark table.dataTable.cell-border>tbody>tr>td{border-top:1px solid rgb(64, 67, 70);border-right:1px solid rgb(64, 67, 70)}html.dark table.dataTable.cell-border>tbody>tr>th:first-child,html.dark table.dataTable.cell-border>tbody>tr>td:first-child{border-left:1px solid rgb(64, 67, 70)}html.dark .dataTables_wrapper .dataTables_filter input,html.dark .dataTables_wrapper .dataTables_length select{border:1px solid rgba(255, 255, 255, 0.2);background-color:var(--dt-html-background)}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.current,html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.current:hover{border:1px solid rgb(89, 91, 94);background:rgba(255, 255, 255, 0.15)}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.disabled,html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover,html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active{color:#666 !important}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button:hover{border:1px solid rgb(53, 53, 53);background:rgb(53, 53, 53)}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button:active{background:#3a3a3a} diff --git a/docs/site_libs/dt-core-1.13.6/js/jquery.dataTables.min.js b/docs/site_libs/dt-core-1.13.6/js/jquery.dataTables.min.js new file mode 100644 index 00000000..f786b0da --- /dev/null +++ b/docs/site_libs/dt-core-1.13.6/js/jquery.dataTables.min.js @@ -0,0 +1,4 @@ +/*! DataTables 1.13.6 + * ©2008-2023 SpryMedia Ltd - datatables.net/license + */ +!function(n){"use strict";var a;"function"==typeof define&&define.amd?define(["jquery"],function(t){return n(t,window,document)}):"object"==typeof exports?(a=require("jquery"),"undefined"==typeof window?module.exports=function(t,e){return t=t||window,e=e||a(t),n(e,t,t.document)}:n(a,window,window.document)):window.DataTable=n(jQuery,window,document)}(function(P,j,v,H){"use strict";function d(t){var e=parseInt(t,10);return!isNaN(e)&&isFinite(t)?e:null}function l(t,e,n){var a=typeof t,r="string"==a;return"number"==a||"bigint"==a||!!h(t)||(e&&r&&(t=$(t,e)),n&&r&&(t=t.replace(q,"")),!isNaN(parseFloat(t))&&isFinite(t))}function a(t,e,n){var a;return!!h(t)||(h(a=t)||"string"==typeof a)&&!!l(t.replace(V,"").replace(/ - - + + Machine learning with tidymodels – 1 - Introduction @@ -95,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -190,10 +190,10 @@ - + - + @@ -203,7 +203,7 @@

1 - Introduction

-

Advanced tidymodels

+

Getting More Out of Feature Engineering and Tuning for Machine Learning

@@ -219,13 +219,13 @@

1 - Introduction

Wi-Fi network name

-

TODO-ADD-LATER

+

Posit Conf 2025

Wi-Fi password

-

TODO-ADD-LATER

+

conf2025

@@ -251,7 +251,7 @@

Who are you?

  • You can use the magrittr %>% or base R |> pipe

  • You are familiar with functions from dplyr, tidyr, ggplot2

  • You have exposure to basic statistical concepts

    • -

  • You do not need intermediate or expert familiarity with modeling or ML

    • +

  • You do need intermediate familiarity with modeling or ML

  • You have used some tidymodels packages

  • You have some experience with evaluating statistical models using resampling techniques

    • @@ -265,6 +265,9 @@

    Who are tidymodels?

  • Max Kuhn
    • +

    + our TA today, Edgar Ruiz!

    +
    +

    Many thanks to Davis Vaughan, Julia Silge, David Robinson, Julie Jung, Alison Hill, and Desirée De Leon for their role in creating these materials!

    @@ -299,130 +302,385 @@

    👀

    Tentative plan for this workshop

      -
    • Feature engineering with recipes
    • Model optimization by tuning
      • Grid search
      • Racing
        • -
      • Iterative methods
        • -
      • -
    • Extras (time permitting) -
        -
      • Effect encodings
        • -
      • A case study
      • +
    • Feature engineering with recipes
      • +
    • Postprocessing
      +
      • +
    • Feature selection

    Introduce yourself to your neighbors 👋

    -



    -

    Log in to Posit Cloud (free): TODO-ADD-LATER

    +
    +
    +

    Getting the materials

    +


    +

    If you are using Posit Cloud:

    +

    Log in to Posit Cloud (free): TODO-ADD-LATER

    +


    +

    If you are working locally:

    +
    # local download
+usethis::use_course("tidymodels/workshops", destdir = "some_path")
+
+# or fork via
+usethis::create_from_github("tidymodels/workshops", fork = TRUE)

    Let’s install some packages

    If you are using your own laptop instead of Posit Cloud:

    -
    # Install the packages for the workshop
-pkgs <- 
-  c("bonsai", "Cubist", "doParallel", "earth", "embed", "finetune", 
-    "lightgbm", "lme4", "parallelly", "plumber", "probably", 
-    "ranger", "rpart", "rpart.plot", "rules", "splines2", "stacks", 
-    "text2vec", "textrecipes", "tidymodels", "vetiver")
-
-install.packages(pkgs)
    +
    # Install the packages for the workshop
+pkgs <- 
+  c("almanac", "betacal", "bonsai", "brulee", "C50", "Cubist", "desirability2", 
+    "dimRed", "earth", "embed", "extrasteps", "finetune", "igraph", 
+    "important", "irlba", "kknn", "lightgbm", "lme4", "mirai", "parallelly", 
+    "plumber", "probably", "RANN", "rpart", "RSpectra", "rules", 
+    "splines2", "stacks", "text2vec", "textrecipes", "tidymodels", 
+    "uwot", "vetiver")
+
+install.packages(pkgs)
    +
    +

    Also, you should make sure that you have installed the newest version of a few packages. To check this, you can run:

    +
    +
    rlang::check_installed("tidymodels", version = "1.4.1")
+rlang::check_installed("embed", version = "1.2.0")
    +
    +
    +
    +
    +

    Let’s get started!

    + +
    +
    +

    Load tidymodels

    +

    We’re here to learn more about how to use the more advanced bits of tidymodels for supervised learning. Let’s load the meta-package:

    +
    +
    library(tidymodels)
+#> ── Attaching packages ──────────────────────────── tidymodels 1.4.1 ──
+#> ✔ broom        1.0.9     ✔ rsample      1.3.1
+#> ✔ dials        1.4.2     ✔ tailor       0.1.0
+#> ✔ dplyr        1.1.4     ✔ tidyr        1.3.1
+#> ✔ infer        1.0.9     ✔ tune         2.0.0
+#> ✔ modeldata    1.5.1     ✔ workflows    1.3.0
+#> ✔ parsnip      1.3.3     ✔ workflowsets 1.1.1
+#> ✔ purrr        1.1.0     ✔ yardstick    1.3.2
+#> ✔ recipes      1.3.1
+#> ── Conflicts ─────────────────────────────── tidymodels_conflicts() ──
+#> ✖ purrr::discard() masks scales::discard()
+#> ✖ dplyr::filter()  masks stats::filter()
+#> ✖ dplyr::lag()     masks stats::lag()
+#> ✖ recipes::step()  masks stats::step()
    +
    +
    +
    +

    Resolve naming conflicts

    +

    You might want to run this function to avoid function name conflicts:

    +


    +
    +
    tidymodels_prefer()
    +
    +


    +

    To get more details, use the quiet = FALSE option.

    +
    +
    +

    Data sets

    +

    For illustration, we’ll use a few different data sets today:

    +
      +
    • class_data: a simulated set of data with a 1:10 class imbalance. Two classes, 20 predictors, and 2,000 data points.
      • +
    • leaf_data: a real data set to identify plant species from their leaves. Thirty-two levels, 53 predictors, and 1,907 data points.
      • +
    • hotel_data: a real data set for predicting the average cost per night. Numeric outcome, 27 predictors, and 15,402 data points.
      • +
    +
    +


    +

    Let’s get warmed up with the first data set.

    +
    +
    +
    +

    Imbalanced data

    +

    These data can be loaded from the GitHub repo:

    +


    +
    +
    "https://raw.githubusercontent.com/tidymodels/" |> 
+  paste0("workshops/main/slides/class_data.RData") |> 
+  url() |> 
+  load()
    -

    Also, you should install the newest version of the dials package (version 1.3.0). To check this, you can run:

    +


    +

    The outcome column is class with levels "event" and "no_event". Predictors are "predictor_01" to "predictor_30".

    +
    +
    +

    Your turn

    +

    Let’s warm up by taking 8 minutes to explore the data.

    +


    +

    We’ll ask you to tell us something about these data that might be interesting for modeling.

    -
    rlang::check_installed("dials", version = "1.3.0")
    +
    +
    +
    +08:00 +
    +
    +
    +
    +
    +
    +

    A quick review of tidymodels

    + +
    +
    +

    Data splitting

    +

    One of our first tasks is to split our data into (at a minimum) a training set and a testing set. The rsample package has numerous functions for this, prefixed by initial_.

    +

    Let’s create a 3:1 split of the simulated data and use a stratified random sample (by class):

    +
    +
    set.seed(429)
+sim_split <- initial_split(class_data, prop = 0.75, strata = class)
+sim_split
+#> <Training/Testing/Total>
+#> <1499/501/2000>
+
+sim_train <- training(sim_split)
+sim_test  <- testing(sim_split)
    +
    +
    +

    AML4TD, TMwR

    -
    -

    Hotel Data

    -

    We’ll use data on hotels to predict the cost of a room.

    -

    The data are in the modeldata package. We’ll sample down the data and refactor some columns:

    +
    +

    Data splitting

    -
    +
    -
    library(tidymodels)
-
-# Max's usual settings: 
-tidymodels_prefer()
-theme_set(theme_bw())
-options(
-  pillar.advice = FALSE, 
-  pillar.min_title_chars = Inf
-)
    +
    sim_train |> 
+  ggplot(aes(class)) + 
+  geom_bar()
    +
    +
    +
    +

    +
    +
    +
    -
    +
    -
    data(hotel_rates)
-set.seed(295)
-hotel_rates <- 
-  hotel_rates |> 
-  sample_n(5000) |> 
-  arrange(arrival_date) |> 
-  select(-arrival_date) |> 
-  mutate(
-    company = factor(as.character(company)),
-    country = factor(as.character(country)),
-    agent = factor(as.character(agent))
-  )
    +
    sim_test |> 
+  ggplot(aes(class)) + 
+  geom_bar()
    +
    +
    +
    +

    +
    +
    +
    -
    -

    Hotel date columns

    -
    -
    names(hotel_rates)
-#>  [1] "avg_price_per_room"             "lead_time"                     
-#>  [3] "stays_in_weekend_nights"        "stays_in_week_nights"          
-#>  [5] "adults"                         "children"                      
-#>  [7] "babies"                         "meal"                          
-#>  [9] "country"                        "market_segment"                
-#> [11] "distribution_channel"           "is_repeated_guest"             
-#> [13] "previous_cancellations"         "previous_bookings_not_canceled"
-#> [15] "reserved_room_type"             "assigned_room_type"            
-#> [17] "booking_changes"                "agent"                         
-#> [19] "company"                        "days_in_waiting_list"          
-#> [21] "customer_type"                  "required_car_parking_spaces"   
-#> [23] "total_of_special_requests"      "arrival_date_num"              
-#> [25] "near_christmas"                 "near_new_years"                
-#> [27] "historical_adr"
    +
    +

    Resampling

    +
    +
    +

    We’ll want to get accurate estimates of model performance.

    +

    Let’s use a resampling method to make multiple versions of our data (using 10-fold cross-validation).

    +


    +

    For large amounts of data, a validation set is also a good alternative.

    +
    +
    +
    +

    +
    +
    +
    +
    +

    AML4TD, TMwR

    -
    -

    Data splitting strategy

    - -
    -
    -

    Data Spending

    -

    Let’s split the data into a training set (75%) and testing set (25%) using stratification:

    +
    +

    Resampling

    -
    set.seed(4028)
-hotel_split <- initial_split(hotel_rates, strata = avg_price_per_room)
-
-hotel_train <- training(hotel_split)
-hotel_test <- testing(hotel_split)
    +
    set.seed(523)
+sim_rs <- vfold_cv(sim_train, v = 10, strata = class)
+sim_rs
+#> #  10-fold cross-validation using stratification 
+#> # A tibble: 10 × 2
+#>    splits             id    
+#>    <list>             <chr> 
+#>  1 <split [1348/151]> Fold01
+#>  2 <split [1349/150]> Fold02
+#>  3 <split [1349/150]> Fold03
+#>  4 <split [1349/150]> Fold04
+#>  5 <split [1349/150]> Fold05
+#>  6 <split [1349/150]> Fold06
+#>  7 <split [1349/150]> Fold07
+#>  8 <split [1349/150]> Fold08
+#>  9 <split [1350/149]> Fold09
+#> 10 <split [1350/149]> Fold10
    -
    +
    +

    Resampled data sets

    +
    +
    model_data_1 <- sim_rs |> get_rsplit(1) |> analysis()
+model_data_1 |> count(class)
+#> # A tibble: 2 × 2
+#>   class        n
+#>   <fct>    <int>
+#> 1 event      151
+#> 2 no_event  1197
+
+perf_data_1 <- sim_rs |> get_rsplit(1) |> assessment()
+perf_data_1 |> count(class)
+#> # A tibble: 2 × 2
+#>   class        n
+#>   <fct>    <int>
+#> 1 event       17
+#> 2 no_event   134
    +
    +
    +
    +

    Models via parsnip

    +

    Let’s fit a simple decision tree to the data:

    +
    +
    # Specify what you want
+tree_spec <- decision_tree(mode = "classification")
+
+# Then train:
+tree_fit <- tree_spec |> fit(class ~ ., data = model_data_1)  
+tree_fit
+#> parsnip model object
+#> 
+#> n= 1348 
+#> 
+#> node), split, n, loss, yval, (yprob)
+#>       * denotes terminal node
+#> 
+#>  1) root 1348 151 no_event (0.11201780 0.88798220)  
+#>    2) predictor_27< -2.268929 76  11 event (0.85526316 0.14473684) *
+#>    3) predictor_27>=-2.268929 1272  86 no_event (0.06761006 0.93238994)  
+#>      6) predictor_29>=2.144976 89  25 event (0.71910112 0.28089888)  
+#>       12) predictor_29>=2.599392 45   3 event (0.93333333 0.06666667) *
+#>       13) predictor_29< 2.599392 44  22 event (0.50000000 0.50000000)  
+#>         26) predictor_18>=0.2344964 32  11 event (0.65625000 0.34375000)  
+#>           52) predictor_03< -0.2257182 16   1 event (0.93750000 0.06250000) *
+#>           53) predictor_03>=-0.2257182 16   6 no_event (0.37500000 0.62500000) *
+#>         27) predictor_18< 0.2344964 12   1 no_event (0.08333333 0.91666667) *
+#>      7) predictor_29< 2.144976 1183  22 no_event (0.01859679 0.98140321) *
    +
    +
    +
    +

    Predicting…

    +
    +
    predict(tree_fit, new_data = head(perf_data_1, 4))
+#> # A tibble: 4 × 1
+#>   .pred_class
+#>   <fct>      
+#> 1 event      
+#> 2 event      
+#> 3 event      
+#> 4 no_event
+
+predict(tree_fit, new_data = head(perf_data_1, 4), type = "prob")
+#> # A tibble: 4 × 2
+#>   .pred_event .pred_no_event
+#>         <dbl>          <dbl>
+#> 1      0.855          0.145 
+#> 2      0.933          0.0667
+#> 3      0.855          0.145 
+#> 4      0.0833         0.917
    +
    +
    +
    +

    Augmenting…

    +
    +
    tree_pred <- augment(tree_fit, new_data = perf_data_1)
+tree_pred |> slice(1:5)
+#> # A tibble: 5 × 34
+#>   .pred_class .pred_event .pred_no_event class predictor_01 predictor_02
+#>   <fct>             <dbl>          <dbl> <fct>        <dbl>        <dbl>
+#> 1 event            0.855          0.145  event       -1.64      -1.74   
+#> 2 event            0.933          0.0667 event       -0.846      0.477  
+#> 3 event            0.855          0.145  event        0.551      0.00571
+#> 4 no_event         0.0833         0.917  event       -1.46      -0.335  
+#> 5 event            0.855          0.145  event       -0.194     -0.276  
+#> # ℹ 28 more variables: predictor_03 <dbl>, predictor_04 <dbl>,
+#> #   predictor_05 <dbl>, predictor_06 <dbl>, predictor_07 <dbl>,
+#> #   predictor_08 <dbl>, predictor_09 <dbl>, predictor_10 <dbl>,
+#> #   predictor_11 <dbl>, predictor_12 <dbl>, predictor_13 <dbl>,
+#> #   predictor_14 <dbl>, predictor_15 <dbl>, predictor_16 <dbl>,
+#> #   predictor_17 <dbl>, predictor_18 <dbl>, predictor_19 <dbl>,
+#> #   predictor_20 <dbl>, predictor_21 <dbl>, predictor_22 <dbl>, …
    +
    +
    +
    +

    Performance metrics

    +

    There are many yardstick metrics* for class predictions and probability estimates.

    +

    Let’s make a collection of metrics and then evaluate our model.

    +
    +
    cls_metrics <- metric_set(brier_class, roc_auc, sensitivity, specificity)
+tree_pred |> cls_metrics(truth = class, estimate = .pred_class, .pred_event)
+#> # A tibble: 4 × 3
+#>   .metric     .estimator .estimate
+#>   <chr>       <chr>          <dbl>
+#> 1 sensitivity binary        0.647 
+#> 2 specificity binary        0.985 
+#> 3 brier_class binary        0.0429
+#> 4 roc_auc     binary        0.899
    +
    +
    +

    * … and metrics for regression and others.

    +
    +
    +

    AML4TD

    +
    +
    +
    +

    Recipes and workflows

    +

    Recipes are preprocessors that perform sequential operations on the preductors.

    +
    +


    +

    For example, to center and scale our predictors:

    +
    +
    rec <- 
+  recipe(class ~ ., data = sim_train) |> 
+  step_normalize(all_numeric_predictors())
    +
    +
    +
    +


    +

    A model, a recipe, and other objects can be added to a workflow to have a single object for the whole modeling sequence:

    +
    +
    tree_wflow <- workflow(rec, tree_spec)
    +
    +
    +
    +

    Your turn

    -

    Let’s take some time and investigate the training data. The outcome is avg_price_per_room.

    -

    Are there any interesting characteristics of the data?

    +

    Fit a different type of decision tree, this time:

    +
      +
    • Using the C5.0 engine
      • +
    • Change the minimum number of samples required for splitting to 10.
      • +
    +


    +

    Did performance change much?

    +


    -
    +
    -10:00 +05:00

    Our versions

    -

    R version 4.5.1 (2025-06-13), Quarto (1.7.32)

    -
    +

    R version 4.5.0 (2025-04-11), Quarto (1.7.31)

    +
    @@ -435,6 +693,14 @@

    Our versions

    +almanac +1.0.0 + + +betacal +0.1.0 + + bonsai 0.4.0 @@ -443,36 +709,48 @@

    Our versions

    1.0.9 +brulee +0.5.0.9000 + + +C50 +0.2.0 + + Cubist 0.5.0 +CVST +0.2-3 + + +desirability2 +0.2.0 + + dials 1.4.2 -doParallel -1.0.17 +dimRed +0.2.7 dplyr 1.1.4 -earth -5.3.4 +DRR +0.0.4 -embed -1.1.5 +earth +5.3.4 -finetune -1.2.1 - - -forested -0.2.0 +embed +1.2.0 @@ -490,13 +768,45 @@

    Our versions

    +extrasteps +0.3.0 + + +finetune +1.2.1 + + +forested +0.2.0 + + Formula 1.2-5 - + ggplot2 3.5.2 + +igraph +2.1.4 + + +important +0.0.1.9000 + + +irlba +2.3.5.1 + + +kernlab +0.9-33 + + +kknn +1.4.1 + lattice 0.22-7 @@ -510,9 +820,28 @@

    Our versions

    1.1-37 +mirai +2.5.0 + + modeldata 1.5.1 + + +
    +
    +
    +
    +
    + + + + + + + + @@ -529,21 +858,6 @@

    Our versions

    - -
    packageversion
    parallelly 1.45.1 plotrix 3.8-4
    -
    -
    -
    -
    -
    - - - - - - - - @@ -557,33 +871,37 @@

    Our versions

    + + + + + + + + + + + + - + - + - - - - - - - -
    packageversion
    plumber 1.3.0 1.1.0
    RANN2.6.2
    recipes 1.3.1
    rpart4.1.24
    rsample 1.3.1
    RSpectra0.16-2
    rules 1.0.2
    scales 1.4.0
    splines2 0.5.4
    stacks1.1.1
    text2vec0.6.4
    @@ -600,16 +918,24 @@

    Our versions

    -textrecipes -1.1.0 +stacks +1.1.1 + + +tailor +0.1.0 + + +text2vec +0.6.4 -tibble -3.3.0 +textrecipes +1.1.0 tidymodels -1.3.0 +1.4.1 tidyr @@ -620,18 +946,22 @@

    Our versions

    2.0.0 +uwot +0.2.3 + + vetiver 0.2.5 - + workflows 1.3.0 - + workflowsets 1.1.1 - + yardstick 1.3.2 @@ -642,7 +972,7 @@

    Our versions

    -
    +
    -
    -

    Previously - Data Usage

    +
    +

    More startup!

    -
    set.seed(4028)
-hotel_split <-
-  initial_split(hotel_rates, strata = avg_price_per_room)
-
-hotel_train <- training(hotel_split)
-hotel_test <- testing(hotel_split)
-
-set.seed(472)
-hotel_rs <- vfold_cv(hotel_train, strata = avg_price_per_room)
    -
    -
    -
    -

    Previously - Boosting Model

    -
    -
    hotel_rec <-
-  recipe(avg_price_per_room ~ ., data = hotel_train) |>
-  step_YeoJohnson(lead_time) |>
-  step_dummy_hash(agent,   num_terms = tune("agent hash")) |>
-  step_dummy_hash(company, num_terms = tune("company hash")) |>
-  step_zv(all_predictors())
-
-lgbm_spec <- 
-  boost_tree(trees = tune(), learn_rate = tune(), min_n = tune()) |> 
-  set_mode("regression") |> 
-  set_engine("lightgbm", num_threads = 1)
-
-lgbm_wflow <- workflow(hotel_rec, lgbm_spec)
-
-lgbm_param <-
-  lgbm_wflow |>
-  extract_parameter_set_dials() |>
-  update(`agent hash`   = num_hash(c(3, 8)),
-         `company hash` = num_hash(c(3, 8)))
    +
    # Load our example data for this section
+"https://raw.githubusercontent.com/tidymodels/" |> 
+  paste0("workshops/main/slides/class_data.RData") |> 
+  url() |> 
+  load()
+
+set.seed(429)
+sim_split <- initial_split(class_data, prop = 0.75, strata = class)
+sim_train <- training(sim_split)
+sim_test  <- testing(sim_split)
+
+set.seed(523)
+sim_rs <- vfold_cv(sim_train, v = 10, strata = class)

    First, a shameless promotion

    +
    +

    Making Grid Search More Efficient

    -

    In the last section, we evaluated 250 models (25 candidates times 10 resamples).

    +

    Previously, we evaluated 250 models (25 candidates times 10 resamples).

    We can make this go faster using parallel processing.

    -

    Also, for some models, we can fit far fewer models than the number that are being evaluated.

    +


    +

    Also, for some models, we can fit far fewer models than the number being evaluated.

      -
    • For boosting, a model with X trees can often predict on candidates with less than X trees.
      • +
    • For example, with boosted trees, a model with X trees can often predict on candidates with fewer than X trees (i.e., no retraining).
    -

    Both of these methods can lead to enormous speed-ups.

    +

    These strategies can lead to enormous speed-ups.

    Model Racing

    Racing is an old tool that we can use to go even faster.

      -
    1. Evaluate all of the candidate models but only for a few resamples.
      2. -
    2. Determine which candidates have a low probability of being selected.
      3. +
    3. Evaluate all of the candidate models, but only for a few resamples.
      4. +
    4. Determine which candidates have a low probability of being selected (cough, cough, tanh activation, cough).
    5. Eliminate poor candidates.
      6. -
    6. Repeat with next resample (until no more resamples remain)
      7. +
    7. Repeat with next resample (until no more resamples remain).

    This can result in fitting a small number of models.

    +

    It is not an iterative search; it is an adaptive grid search.

    +
    +

    TMwR, TMwR example, AML4TD

    +

    Discarding Candidates

    How do we eliminate tuning parameter combinations?

    There are a few methods to do so. We’ll use one based on analysis of variance (ANOVA).

    -

    However… there is typically a large difference between resamples in the results.

    +

    However… there is typically a large resampling effect in the results.

    Resampling Results (Non-Racing)

    @@ -329,7 +295,7 @@

    Resampling Results (Non-Racing)

    -

    +

    @@ -342,22 +308,22 @@

    Are Candidates Different?

    • or a t-test on their differences matched by resamples
    -

    With \(n = 10\) resamples, the confidence interval for the difference in RMSE is (0.99, 2.8), indicating that candidate number 2 has smaller error.

    +

    With \(n = 10\) resamples, the confidence interval for the difference in the model error is (0.99, 2.8), indicating that candidate number 2 has a smaller error.

    Evaluating Differences in Candidates

    -

    What if we were to have compared the candidates while we seqeuntially evaluated each resample?

    +

    What if we were to have compared the candidates while we sequentially evaluated each resample?

    👉

    -

    One candidate shows superiority when 4 resamples have been evaluated.

    +

    One candidate shows superiority when 5 resamples have been evaluated.

    -

    +

    @@ -372,54 +338,137 @@

    Interim Analysis of Results


    Kuhn (2014) has examples and simulations to show that the method works.

    The finetune package has functions tune_race_anova() and tune_race_win_loss().

    +
    +
    +
    +

    Boosted Trees

    + +
    +
    +

    Boosted Trees

    +

    These are popular ensemble methods that build a sequence of tree models.

    +


    +

    Each tree uses the results of the previous tree to better predict samples, especially those that have been poorly predicted.

    +


    +

    Each tree in the ensemble is saved, and new samples are predicted using a weighted average of its votes.

    +


    +

    We’ll focus on the popular lightgbm implementation.

    +
    +
    +

    Boosted Tree Tuning Parameters

    +

    Some possible parameters:

    +
      +
    • mtry: The number of predictors randomly sampled at each split (in \([1, ncol(x)]\) or \((0, 1]\)).
      • +
    • trees: The number of trees (\([1, \infty]\), but usually up to thousands).
      • +
    • min_n: The number of samples needed to further split (\([1, n]\)).
      • +
    • learn_rate: The rate that each tree adapts from previous iterations (\((0, \infty]\), usual maximum is 0.1).
      • +
    • stop_iter: The number of iterations of boosting where no improvement was shown before stopping (\([1, trees]\)).
      • +
    +
    +
    +

    Boosted Tree Tuning Parameters

    +

    TBH, it is usually not difficult to optimize these models.

    +


    +
    +
    +

    Often, there are multiple candidate tuning parameter regions with very good results.

    +

    For example: 👉

    +


    +

    To demonstrate, we’ll look at optimizing five of the tuning parameters.

    +
    +
    +
    +

    +
    +
    +
    +
    +
    +

    Boosted Tree Tuning Parameters

    +

    We’ll need to load the bonsai package. This has the information needed to use lightgbm

    +
    +
    library(bonsai)
+
+lgbm_spec <-
+  boost_tree(
+    trees = tune(),
+    learn_rate = tune(),
+    mtry = tune(),
+    min_n = tune(),
+    stop_iter = tune()
+  ) |>
+  set_mode("classification") |>
+  # Turn off within-tree parallel processing; it's faster to run 
+  # the resamples/configurations in parallel
+  set_engine("lightgbm", num_threads = 1) 
+
+# No preprocessing required:
+lgbm_wflow <- workflow(class ~ ., lgbm_spec)
    +
    +
    +
    +
    +

    Racing our boosted trees

    +
    -

    Racing

    +

    Racing

    -
    # Let's use a larger grid
-lgbm_grid <- 
-  lgbm_param |> 
-  grid_space_filling(size = 50)
-
-library(finetune)
-
-set.seed(9)
-lgbm_race_res <-
-  lgbm_wflow |>
-  tune_race_anova(
-    resamples = hotel_rs,
-    grid = lgbm_grid, 
-    metrics = reg_metrics
-  )
    +
    library(finetune)
+
+# Set this to true to demo
+ctrl <- control_race(verbose_elim = FALSE)
+
+# Optimizes on the first metric in the set
+cls_mtr <- metric_set(brier_class, roc_auc, sensitivity, specificity)
+
+mirai::daemons(parallel::detectCores() - 1)
+
+set.seed(321)
+lgbm_res <-
+  lgbm_wflow |>
+  tune_race_anova(              # <- very similar syntax to tune_grid()
+    resamples = sim_rs,
+    # Let's use a larger grid
+    grid = 50,
+    control = ctrl,
+    metrics = cls_mtr
+  )
    -

    The syntax and helper functions are extremely similar to those shown for tune_grid().

    Racing Results

    -
    show_best(lgbm_race_res, metric = "mae")
-#> # A tibble: 2 × 11
-#>   trees min_n learn_rate `agent hash` `company hash` .metric .estimator  mean     n std_err .config          
-#>   <int> <int>      <dbl>        <int>          <int> <chr>   <chr>      <dbl> <int>   <dbl> <chr>            
-#> 1  1347     5     0.0655           66             26 mae     standard    9.64    10   0.173 pre31_mod34_post0
-#> 2   980     8     0.0429           17            135 mae     standard    9.76    10   0.164 pre12_mod25_post0
    +
    show_best(lgbm_res, metric = "brier_class")
+#> # A tibble: 1 × 11
+#>    mtry trees min_n learn_rate stop_iter .metric .estimator   mean     n std_err
+#>   <int> <int> <int>      <dbl>     <int> <chr>   <chr>       <dbl> <int>   <dbl>
+#> 1     9  1836     9    0.00222         6 brier_… binary     0.0379    10 0.00238
+#> # ℹ 1 more variable: .config <chr>
    +
    +


    +
    +

    Times using 10 cores: sequential: 605s, parallel: 92s, and parallel racing: 50s.

    +


    +
    +
    +

    Parallel was 6.6-fold faster, and racing in parallel was 12.3-fold faster.

    Racing Results

    -

    Only 171 models were fit (out of 500).

    +

    Only 378 models were fit (out of 500).

    select_best() never considers candidate models that did not get to the end of the race.

    There is a helper function to see how candidate models were removed from consideration.

    -
    plot_race(lgbm_race_res) + 
-  scale_x_continuous(breaks = pretty_breaks())
    +
    plot_race(lgbm_res)
    -

    +

    @@ -429,20 +478,20 @@

    Racing Results

    Your turn

      -
    • Run tune_race_anova() with a different seed.
      • +
    • Run tune_race_anova() with a different seed and/or a different metric.
    • Did you get the same or similar results?
    -10:00 +08:00
    -
    +
    @@ -11575,190 +11576,172 @@

    isomap with recipes + }); + \ No newline at end of file diff --git a/docs/slides/advanced-04-feature-engineering-part-one_files/figure-revealjs/umap-known-1.svg b/docs/slides/advanced-04-feature-engineering-part-one_files/figure-revealjs/umap-known-1.svg index afb4b314..0ddcaa17 100644 --- a/docs/slides/advanced-04-feature-engineering-part-one_files/figure-revealjs/umap-known-1.svg +++ b/docs/slides/advanced-04-feature-engineering-part-one_files/figure-revealjs/umap-known-1.svg @@ -35,123 +35,119 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -163,129 +159,128 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -297,125 +292,118 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -427,127 +415,127 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -595,86 +583,70 @@ 3 - - - - - --2 --1 -0 -1 -2 - - - - - --2 --1 -0 -1 -2 - - - - --10 --5 -0 -5 - - - - - - --2 --1 -0 -1 -2 -3 --2 -0 -2 - - - --2 --1 -0 -1 -2 - - - - - --10 --5 -0 -5 -10 - - - - - --3 --2 --1 -0 -1 -2 -3 - - - - - - - + + + + + +-2 +-1 +0 +1 +2 + + + + + +-2 +-1 +0 +1 +2 + + + +-5 +0 +5 + + + +-2 +0 +2 +-2 +0 +2 + + + +-2 +-1 +0 +1 +2 + + + + + +-10 +0 +10 + + + +-2 +-1 +0 +1 +2 + + + + + UMAP1 UMAP2 diff --git a/docs/slides/advanced-04-feature-engineering-part-one_files/figure-revealjs/umap-unknown-1.svg b/docs/slides/advanced-04-feature-engineering-part-one_files/figure-revealjs/umap-unknown-1.svg index 0143a62f..f633f66b 100644 --- a/docs/slides/advanced-04-feature-engineering-part-one_files/figure-revealjs/umap-unknown-1.svg +++ b/docs/slides/advanced-04-feature-engineering-part-one_files/figure-revealjs/umap-unknown-1.svg @@ -35,123 +35,119 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -163,129 +159,128 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -297,125 +292,118 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -427,127 +415,127 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -595,86 +583,70 @@ B - - - - - --2 --1 -0 -1 -2 - - - - - --2 --1 -0 -1 -2 - - - - --10 --5 -0 -5 - - - - - - --2 --1 -0 -1 -2 -3 --2 -0 -2 - - - --2 --1 -0 -1 -2 - - - - - --10 --5 -0 -5 -10 - - - - - --3 --2 --1 -0 -1 -2 -3 - - - - - - - + + + + + +-2 +-1 +0 +1 +2 + + + + + +-2 +-1 +0 +1 +2 + + + +-5 +0 +5 + + + +-2 +0 +2 +-2 +0 +2 + + + +-2 +-1 +0 +1 +2 + + + + + +-10 +0 +10 + + + +-2 +-1 +0 +1 +2 + + + + + UMAP1 UMAP2 diff --git a/docs/slides/advanced-04-feature-engineering-part-two.html b/docs/slides/advanced-04-feature-engineering-part-two.html deleted file mode 100644 index 86ee0693..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two.html +++ /dev/null @@ -1,1658 +0,0 @@ - - - - - - - - - - - - - Machine learning with tidymodels – 6 - Feature engineering: splines, target encoding and dates - - - - - - - - - - - - - - - - - - - - - - - -
    -
    - -
    -

    6 - Feature engineering: splines, target encoding and dates

    -

    Getting More Out of Feature Engineering and Tuning for Machine Learning

    - -
    -
    - -
    -
    -

    Getting set up

    -
    -
    library(tidymodels)
-library(embed)
-library(extrasteps)
-
-tidymodels_prefer()
-theme_set(theme_bw())
-options(pillar.advice = FALSE, pillar.min_title_chars = Inf)
    -
    -
    -
    -
    -

    Hotel data

    - -
    -
    -

    Hotel rates data set

    -

    Regression data set for predicting the average daily rate for a room, for “Resort Hotel”. The agent and company use random names.

    -
    -
    glimpse(hotel_rates)
-#> Rows: 15,402
-#> Columns: 28
-#> $ avg_price_per_room             <dbl> 110.00, 74.00, 81.90, 81.00, 112.20, 90…
-#> $ lead_time                      <dbl> 241, 273, 248, 236, 243, 267, 94, 10, 1…
-#> $ stays_in_weekend_nights        <dbl> 0, 2, 2, 2, 4, 2, 4, 0, 0, 0, 0, 0, 0, …
-#> $ stays_in_week_nights           <dbl> 1, 5, 5, 5, 10, 5, 7, 1, 1, 1, 1, 1, 1,…
-#> $ adults                         <dbl> 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, …
-#> $ children                       <dbl> 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, …
-#> $ babies                         <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …
-#> $ meal                           <fct> bed_and_breakfast, bed_and_breakfast, b…
-#> $ country                        <fct> prt, aus, gbr, prt, gbr, null, prt, esp…
-#> $ market_segment                 <fct> online_travel_agent, offline_travel_age…
-#> $ distribution_channel           <fct> ta_to, ta_to, ta_to, ta_to, ta_to, ta_t…
-#> $ is_repeated_guest              <dbl> 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, …
-#> $ previous_cancellations         <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …
-#> $ previous_bookings_not_canceled <dbl> 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, …
-#> $ reserved_room_type             <fct> a, a, a, a, a, a, f, e, h, a, a, g, a, …
-#> $ assigned_room_type             <fct> c, a, c, a, a, a, f, f, h, e, e, g, e, …
-#> $ booking_changes                <dbl> 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, …
-#> $ agent                          <fct> devin_rivera_borrego, lia_nauth, jawhar…
-#> $ company                        <fct> not_applicable, not_applicable, not_app…
-#> $ days_in_waiting_list           <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …
-#> $ customer_type                  <fct> transient, transient_party, transient, …
-#> $ required_car_parking_spaces    <dbl> 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, …
-#> $ total_of_special_requests      <dbl> 1, 0, 0, 2, 0, 0, 1, 1, 0, 2, 2, 0, 2, …
-#> $ arrival_date                   <date> 2016-07-02, 2016-07-02, 2016-07-02, 20…
-#> $ arrival_date_num               <dbl> 2016.5, 2016.5, 2016.5, 2016.5, 2016.5,…
-#> $ near_christmas                 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …
-#> $ near_new_years                 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, …
-#> $ historical_adr                 <dbl> 104.9811, 104.9811, 104.9811, 104.9811,…
    -
    -
    -

    Antonio, N., de Almeida, A., and Nunes, L. (2019). Hotel booking demand datasets. Data in Brief, 22, 41-49.

    -
    -
    -
    -

    Hotel data splitting

    -

    Generally, you should always do data splitting. We are doing it here explicitly because some artifacts of splitting data become useful later on.

    -
    -
    set.seed(1234)
-hotel_split <- initial_split(hotel_rates)
-hotel_train <- training(hotel_split)
-hotel_test <- testing(hotel_split)
    -
    -
    -
    -

    Your turn

    -

    -

    Load and explore the hotel_train data

    -

    Comes loaded with the modeldata package

    -
    -
    -
    -
    -05:00 -
    -
    -
    -
    -
    -

    Nonlinear predictors

    - -

    -Figure 1 -

    -
    -
    -

    Splines

    - -
    -
    -

    Splines

    -

    It is a way to transform a single numeric predictor into multiple numeric predictors, with the hope that the new numeric predictors are more linearly related to the outcome.

    -

    Mostly needed with linear models, but it should rarely hurt to use it.

    -
    -

    FEAZ FES

    -
    -
    -
    -

    Splines explained

    -

    A B-spline is a piecewise polynomial function.

    -

    We have 2 main parameters to worry about. Number of knots and the polynomial degree.

    -

    The domain of the predictor is split into k regions, with a knot between each, and a polynomial function is fit within each region, under the constraint that they touch each other at the knot.

    -
    -
    -

    knots: 1, degree: 1

    - -

    -Figure 2 -

    -
    -

    knots: 2, degree: 1

    - -

    -Figure 3 -

    -
    -

    knots: 5, degree: 1

    - -

    -Figure 4 -

    -
    -

    knots: 5, degree: 2

    - -

    -Figure 5 -

    -
    -

    knots: 5, degree: 3

    - -

    -Figure 6 -

    -
    -

    knots: 9, degree: 3

    - -

    -Figure 7 -

    -
    -

    B-Spline features visualized - degree: 3

    - -
    -
    -

    Splines as numbers

    -
    -
    - --------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    arrival_date_numSpline Feature 1Spline Feature 2Spline Feature 3Spline Feature 4Spline Feature 5Spline Feature 6
    2017.6190.000.000.000.030.350.62
    2016.8440.150.590.260.000.000.00
    2016.7020.510.400.050.000.000.00
    2017.0770.000.190.670.140.000.00
    2016.8610.130.580.290.000.000.00
    2017.0190.000.300.620.070.000.00
    2017.1230.000.110.660.230.000.00
    -
    -
    -
    -
    -

    Splines pros and cons

    -
    -
    -

    Pros

    -
      -
    • fast
      • -
    • easy to use
      • -
    • semi-interpretable
      • -
    -
    -

    Cons

    -
      -
    • adds more columns
      • -
    • will need to select the number of columns
      • -
    • can be messy outside the range
      • -
    -
    -
    -
    -

    Your turn

    -

    -

    Apply B-splines to some variables using step_spline_b()

    -
    -
    -
    -
    -03:00 -
    -
    -
    -
    -
    -

    Factors with many categories

    -
    -
    -
    -
    hotel_train |>
-  count(country)
-#> # A tibble: 88 × 2
-#>    country     n
-#>    <fct>   <int>
-#>  1 ago         7
-#>  2 and         1
-#>  3 are         3
-#>  4 arg        14
-#>  5 aus        31
-#>  6 aut        76
-#>  7 aze         2
-#>  8 bel       184
-#>  9 bgr         2
-#> 10 bhs         1
-#> # ℹ 78 more rows
    -
    -
    -
    -
    hotel_train |>
-  count(company)
-#> # A tibble: 157 × 2
-#>    company                 n
-#>    <fct>               <int>
-#>  1 abdou_llc               2
-#>  2 afework_llc             5
-#>  3 alston_pbc              3
-#>  4 battle_llc             23
-#>  5 bennett_and_company     3
-#>  6 berhanu_pbc            53
-#>  7 biggers_llc             4
-#>  8 blasingime_llc          1
-#>  9 boddy_llc              16
-#> 10 boles_pbc               3
-#> # ℹ 147 more rows
    -
    -
    -
    -
    hotel_train |>
-  count(agent)
-#> # A tibble: 119 × 2
-#>    agent                 n
-#>    <fct>             <int>
-#>  1 aaron_marquez         2
-#>  2 alexander_drake    1117
-#>  3 allen_her             1
-#>  4 anas_el_bashir        1
-#>  5 araseli_billy         1
-#>  6 arhab_al_islam        7
-#>  7 audray_tucker        38
-#>  8 bernice_baltierra    35
-#>  9 betzy_rodriguez      66
-#> 10 brayan_guerrero       2
-#> # ℹ 109 more rows
    -
    -
    -
    -
    -

    How do we handle them?

    -
    -

    We could:

    -
      -

    • Make the full set of indicator variables 😳

      • -

    • Lump agents and companies that rarely occur into an “other” group

      • -

    • Use feature hashing to create a smaller set of indicator variables

      • -

    • Use target encoding to replace the county, agent, and company columns with the estimated effect of that predictor

      • -
    -
    -
    -
    -
    -

    Target encoding

    - -
    -
    -

    Target encoding

    -

    Target encoding (also called mean encoding, likelihood encoding, impact encoding, or effect encoding) is a supervised trained method that turns a single categorical predictor into a single numeric predictor.

    -

    It is often used to deal with categorical predictors with many levels, although it works regardless.

    -

    Since it uses the outcome to train it, you need to make sure to use cross-validation to avoid overfitting.

    -
    -

    FEAZ FES

    -
    -
    -
    -

    Target encoding motivation

    -

    You have a numeric outcome and a categorical predictor. And you want to transform each value of the categorical predictor into a value that best represents the outcome?

    -
    -
    -
    -

    We calculate the mean of the outcome within each level of the predictor, and use that as the new value.

    -
    -
    -
    -
    - -
    -

    Caution

    -
    -
    -

    Don’t do just this! We are building up the method one thing at a time. Unregularized target encoding is really prone to overfitting.

    -
    -
    -
    -
    -
    -
    hotel_train |>
-  summarise(
-    mean = mean(avg_price_per_room),
-    .by = agent
-  )
-#> # A tibble: 119 × 2
-#>    agent                 mean
-#>    <fct>                <dbl>
-#>  1 alexander_drake      144. 
-#>  2 kaylae_maxedon        62.5
-#>  3 michael_mcdole        60.9
-#>  4 devin_rivera_borrego 126. 
-#>  5 james_richards        78.6
-#>  6 estela_bonilla        41.9
-#>  7 charles_najera       109. 
-#>  8 reema_el_tamer       118. 
-#>  9 jawhara_al_azad       90.1
-#> 10 not_applicable        84.1
-#> # ℹ 109 more rows
    -
    -
    -
    -
    -
    -

    Target encoding handling unseen levels

    -
    -
    -
    -
    - -
    -

    Caution

    -
    -
    -

    Don’t look at the testing data set. This is done for educational purposes.

    -
    -
    -
    -
    -
    -
    -
    hotel_train |>
-  count(agent, .drop = FALSE)
-#> # A tibble: 174 × 2
-#>    agent                   n
-#>    <fct>               <int>
-#>  1 aaron_marquez           2
-#>  2 aayaat_al_farran        0
-#>  3 alanah_cook             0
-#>  4 alexander_drake      1117
-#>  5 allen_her               1
-#>  6 amirah_christian        0
-#>  7 anas_el_bashir          1
-#>  8 anna_beltran_moreno     0
-#>  9 anna_choi               0
-#> 10 araseli_billy           1
-#> # ℹ 164 more rows
    -
    -
    -
    -
    hotel_test |>
-  count(agent, .drop = FALSE)
-#> # A tibble: 174 × 2
-#>    agent                   n
-#>    <fct>               <int>
-#>  1 aaron_marquez           1
-#>  2 aayaat_al_farran        0
-#>  3 alanah_cook             0
-#>  4 alexander_drake       367
-#>  5 allen_her               1
-#>  6 amirah_christian        0
-#>  7 anas_el_bashir          0
-#>  8 anna_beltran_moreno     0
-#>  9 anna_choi               0
-#> 10 araseli_billy           1
-#> # ℹ 164 more rows
    -
    -
    -
    -
    -

    Target encoding handling unseen levels

    -
    -
    -

    Calculate the global mean of the outcome and use it for cases that aren’t seen in the training data set.

    -


    -
    -
    mean(hotel_train$avg_price_per_room)
-#> [1] 104.6039
    -
    -
    -
    -
    hotel_train |>
-  summarise(
-    mean = mean(avg_price_per_room),
-    .by = agent
-  )
-#> # A tibble: 119 × 2
-#>    agent                 mean
-#>    <fct>                <dbl>
-#>  1 alexander_drake      144. 
-#>  2 kaylae_maxedon        62.5
-#>  3 michael_mcdole        60.9
-#>  4 devin_rivera_borrego 126. 
-#>  5 james_richards        78.6
-#>  6 estela_bonilla        41.9
-#>  7 charles_najera       109. 
-#>  8 reema_el_tamer       118. 
-#>  9 jawhara_al_azad       90.1
-#> 10 not_applicable        84.1
-#> # ℹ 109 more rows
    -
    -
    -
    -
    -

    How do we handle low counts?

    -
    -
    -

    Some of the levels have very low counts. We can’t have the same confidence in those means as the means calculated on high counts.

    -

    We use the global mean to account for 0 occurrences. Let us adjust the calculated mean by the global mean depending on the counts.

    -
    -
    -
    hotel_train |>
-  summarise(
-    mean = mean(avg_price_per_room),
-    n = n(),
-    .by = agent
-  ) |>
-  arrange(agent)
-#> # A tibble: 119 × 3
-#>    agent              mean     n
-#>    <fct>             <dbl> <int>
-#>  1 aaron_marquez     118.      2
-#>  2 alexander_drake   144.   1117
-#>  3 allen_her          65       1
-#>  4 anas_el_bashir     99       1
-#>  5 araseli_billy      40       1
-#>  6 arhab_al_islam     35       7
-#>  7 audray_tucker      76.0    38
-#>  8 bernice_baltierra  71.3    35
-#>  9 betzy_rodriguez    84.0    66
-#> 10 brayan_guerrero    37.5     2
-#> # ℹ 109 more rows
    -
    -
    -
    -
    -

    Partial pooling

    -

    Partial pooling somewhat lowers the risk of overfitting since it tends to correct for agents with small sample sizes. It can’t correct for improper data usage or data leakage, though.

    -
    -
    -

    Partial pooling results

    - -

    -Figure 8 -

    -
    -

    Implentations

    -

    We have described this method solely based on analytical calculations (step_lencode()), but you could arrive at similar numbers using a model-based approach by fitting a no-intercept generalized linear model. A hierarchical version would induce partial pooling.

    -
      -
    • step_lencode_glm()
      • -
    • step_lencode_bayes()
      • -
    • step_lencode_mixed()
      • -
    -
    -
    -

    Target encoding in recipes

    -
    -
    recipe(avg_price_per_room ~ ., data = hotel_train) |>
-  step_lencode(
-    agent, country, company,
-    outcome = vars("avg_price_per_room"), smooth = TRUE,
-  ) |>
-  prep() |>
-  bake(NULL) |>
-  select(agent, country, company)
-#> # A tibble: 11,551 × 3
-#>    agent country company
-#>    <dbl>   <dbl>   <dbl>
-#>  1 144.    108.     109.
-#>  2  77.2    83.4    109.
-#>  3  61.5    99.9    109.
-#>  4 126.    108.     109.
-#>  5 144.    108.     109.
-#>  6  79.9    73.8    109.
-#>  7 126.     99.9    109.
-#>  8  69.8   108.     109.
-#>  9 126.     99.9    109.
-#> 10 144.    108.     109.
-#> # ℹ 11,541 more rows
    -
    -
    -
    -

    Your turn

    -

    -

    Apply target encoding to the data set, see how it affects different predictors, not just the ones we listed here.

    -
      -
    • step_lencode()
      • -
    • step_lencode_glm()
      • -
    • step_lencode_bayes()
      • -
    • step_lencode_mixed()
      • -
    -
    -
    -
    -
    -03:00 -
    -
    -
    -
    -
    -
    -

    Date time variables

    - -
    -
    -

    Date time variables

    -

    How can we represent the date column arrival_date for our model?

    -
    -

    When we use a date column in its native format, most models in R convert it to an integer.

    -
    -
    -

    We can re-engineer it as:

    -
      -
    • Days since a reference date
      • -
    • Day of the week
      • -
    • Month
      • -
    • Year
      • -
    • Indicators for holidays
      • -
    - -
    -

    FEAZ

    -
    -
    -
    -
    -

    Your turn

    -

    -

    Explore the arrival_date variable and its relation to avg_price_per_room

    -

    The lubridate package might provide helpful

    -
    -
    -
    -
    -05:00 -
    -
    -
    -
    -
    -

    arrival_date date features

    -

    using step_date( features = c("year", "month", "dow", "decimal", "mday", "doy", "week", "semester", "quarter"))

    -
    -
    - ------------ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    arrival_dateyearmonthdowdecimalmdaydoyweeksemesterquarter
    2016-08-302016AugTue2016.661302433523
    2016-10-222016OctSat2016.806222964324
    2016-12-172016DecSat2016.959173525124
    2017-02-132017FebMon2017.1181344711
    2017-04-052017AprWed2017.2585951412
    -
    -
    -
    -
    -

    arrival_date date features

    -

    Adding label = FALSE

    -
    -
    - ------------ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    arrival_dateyearmonthdowdecimalmdaydoyweeksemesterquarter
    2016-08-302016832016.661302433523
    2016-10-2220161072016.806222964324
    2016-12-1720161272016.959173525124
    2017-02-132017222017.1181344711
    2017-04-052017442017.2585951412
    -
    -
    -
    -
    -

    Other recipes steps

    -
      -
    • step_time()
      • -
    -

    Works the same as step_date() but for measurements smaller than day: hour, hour12, am/pm, minute, second, decimal_day.

    -
      -
    • step_holiday()
      • -
    -

    Adds indicators for holidays. See timeDate::listHolidays() for supported holidays.

    -
    -
    -

    Your turn

    -

    -

    Apply date steps to the arrival_date variable and try to see if we capture anything about avg_price_per_room

    -
    -
    -
    -
    -03:00 -
    -
    -
    -
    -
    -

    dates as numerics

    - -

    -Figure 9 -

    -
    -

    holidays as numerics

    - -

    -Figure 10 -

    -
    -

    What are the issues with these features?

    -

    The numeric features make it easy to capture the end or beginning, but harder to do anything more granular.

    -

    The indicators mostly care about the day itself. No information about the lead-up or aftermath

    -
    -
    -

    time events

    -

    Using extrasteps::step_time_event() and the almanac package, we can create useful time features.

    -
    -
    library(almanac)
-
-rule_1 <- weekly() |>
-  recur_on_weekdays() |>
-  rsetdiff(hol_christmas())
-
-rule_2 <- monthly(since = "2000-01-01") |>
-  recur_on_interval(3) |>
-  recur_on_day_of_month(1)
-
-rule_3 <- yearly("1997-06-05") |>
-  recur_on_day_of_week("Thursday") |>
-  recur_on_month_of_year(c("Jun", "July", "Aug"))
    -
    -
    -
    -

    step_time_event()

    -

    Create a list of rules (last slide) and pass them to the rules argument of step_time_event()

    -
    -
    rules <- list(rule_1 = rule_1, rule_2 = rule_2, rule_3 = rule_3)
-
-recipe(~arrival_date, data = hotel_rates) |>
-  step_time_event(arrival_date, rules = rules)
    -
    -
    -

    FEAZ

    -
    -
    -
    -

    step_time_event() as numerics

    -
    -
    -
    -
    -
    - -
    -
    -Figure 11 -
    -
    -
    -
    -
    -
    -
    -

    Non-indicator time events

    -

    These features still have the issue that they only attach value to the date itself.

    -

    We can attach values based on how far we are away from those dates.

    -
      -
    • step_date_before()
      • -
    • step_date_after()
      • -
    • step_date_nearest()
      • -
    -
    -
    -

    step_date_before()

    - -

    -Figure 12 -

    -
    -

    step_date_before() - inverse

    - -

    -Figure 13 -

    -
    -

    step_date_after() - inverse

    - -

    -Figure 14 -

    -
    -

    step_date_nearest() - inverse

    - -

    -Figure 15 -

    -
    -

    Datetime features

    -

    Avoid crafting datetime features by hand if at all possible.

    -

    Dealing with uneven month lengths, leap days (leap seconds)

    -

    Or tried to define any event that doesn’t land on the same day of the week or date each year.

    -
    -

    The first Sunday after the first full moon on or after the vernal equinox

    -
    - - -
    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/data-splines-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/data-splines-1.svg deleted file mode 100644 index bdf31ff2..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/data-splines-1.svg +++ /dev/null @@ -1,292 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Spline Feature 4 - - - - - - - - - -Spline Feature 5 - - - - - - - - - -Spline Feature 6 - - - - - - - - - -Spline Feature 1 - - - - - - - - - -Spline Feature 2 - - - - - - - - - -Spline Feature 3 - - -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -0.00 -0.25 -0.50 -0.75 -1.00 -0.00 -0.25 -0.50 -0.75 -1.00 -arrival_date_num -value - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-arrival-date-num-vs-avg-price-per-room-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-arrival-date-num-vs-avg-price-per-room-1.svg deleted file mode 100644 index c1a80e25..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-arrival-date-num-vs-avg-price-per-room-1.svg +++ /dev/null @@ -1,11635 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -100 -200 -300 -400 - - - - - - - - - - -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -arrival_date_num -avg_price_per_room - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-partial-pooling-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-partial-pooling-1.svg deleted file mode 100644 index bb0a9cec..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-partial-pooling-1.svg +++ /dev/null @@ -1,189 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -50 -75 -100 -125 -150 -175 - - - - - - - - - - -50 -100 -150 -200 -mean -partial pooled mean - -n - - - - - - -1000 -2000 -3000 - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-1-degree-1-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-1-degree-1-1.svg deleted file mode 100644 index 573b2dea..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-1-degree-1-1.svg +++ /dev/null @@ -1,11623 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -100 -200 -300 -400 -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -arrival_date_num -outcome - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-2-degree-1-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-2-degree-1-1.svg deleted file mode 100644 index 7e26400e..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-2-degree-1-1.svg +++ /dev/null @@ -1,11623 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -100 -200 -300 -400 -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -arrival_date_num -outcome - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-5-degree-1-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-5-degree-1-1.svg deleted file mode 100644 index 518a55d0..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-5-degree-1-1.svg +++ /dev/null @@ -1,11623 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -100 -200 -300 -400 -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -arrival_date_num -outcome - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-5-degree-2-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-5-degree-2-1.svg deleted file mode 100644 index fd9b85b0..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-5-degree-2-1.svg +++ /dev/null @@ -1,11623 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -100 -200 -300 -400 -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -arrival_date_num -outcome - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-5-degree-3-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-5-degree-3-1.svg deleted file mode 100644 index a3bd4ad2..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-5-degree-3-1.svg +++ /dev/null @@ -1,11623 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -100 -200 -300 -400 -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -arrival_date_num -outcome - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-9-degree-3-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-9-degree-3-1.svg deleted file mode 100644 index 0b121234..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-spline-knots-9-degree-3-1.svg +++ /dev/null @@ -1,11623 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -100 -200 -300 -400 -2016.50 -2016.75 -2017.00 -2017.25 -2017.50 -arrival_date_num -outcome - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-after-inverse-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-after-inverse-1.svg deleted file mode 100644 index 577a3ac0..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-after-inverse-1.svg +++ /dev/null @@ -1,1475 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -after_rule_1 - - - - - - - - - - -after_rule_2 - - - - - - - - - - -after_rule_3 - - - - - - - -Jul 2016 -Oct 2016 -Jan 2017 -Apr 2017 -Jul 2017 -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -arrival_date -value - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-before-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-before-1.svg deleted file mode 100644 index a74b3649..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-before-1.svg +++ /dev/null @@ -1,1465 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -before_rule_1 - - - - - - - - - - -before_rule_2 - - - - - - - - - - -before_rule_3 - - - - - - - -Jul 2016 -Oct 2016 -Jan 2017 -Apr 2017 -Jul 2017 -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -0 -25 -50 -75 - - - - -0 -100 -200 - - - -arrival_date -value - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-before-inverse-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-before-inverse-1.svg deleted file mode 100644 index 1eeec3c3..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-before-inverse-1.svg +++ /dev/null @@ -1,1475 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -before_rule_1 - - - - - - - - - - -before_rule_2 - - - - - - - - - - -before_rule_3 - - - - - - - -Jul 2016 -Oct 2016 -Jan 2017 -Apr 2017 -Jul 2017 -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -arrival_date -value - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-nearest-inverse-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-nearest-inverse-1.svg deleted file mode 100644 index 20b4e840..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-nearest-inverse-1.svg +++ /dev/null @@ -1,1475 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -nearest_rule_1 - - - - - - - - - - -nearest_rule_2 - - - - - - - - - - -nearest_rule_3 - - - - - - - -Jul 2016 -Oct 2016 -Jan 2017 -Apr 2017 -Jul 2017 -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -0.0 -0.5 -1.0 -1.5 -2.0 - - - - - -arrival_date -value - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-holiday-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-holiday-1.svg deleted file mode 100644 index e99fef78..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-holiday-1.svg +++ /dev/null @@ -1,1475 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -ChristmasDay - - - - - - - - - - -LaborDay - - - - - - - - - - -NewYearsDay - - - - - - - -Jul 2016 -Oct 2016 -Jan 2017 -Apr 2017 -Jul 2017 -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - -arrival_date -value - - - diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-time-event-1.svg b/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-time-event-1.svg deleted file mode 100644 index 726ab86c..00000000 --- a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-time-event-1.svg +++ /dev/null @@ -1,1475 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -rule_1 - - - - - - - - - - -rule_2 - - - - - - - - - - -rule_3 - - - - - - - -Jul 2016 -Oct 2016 -Jan 2017 -Apr 2017 -Jul 2017 -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - -0.00 -0.25 -0.50 -0.75 -1.00 - - - - - -arrival_date -value - - - diff --git a/docs/slides/advanced-05-feature-engineering-part-one.html b/docs/slides/advanced-05-feature-engineering-part-one.html deleted file mode 100644 index 125c407a..00000000 --- a/docs/slides/advanced-05-feature-engineering-part-one.html +++ /dev/null @@ -1,11764 +0,0 @@ - - - - - - - - - - - - - Machine learning with tidymodels – 5 - Feature engineering: dummies and embeddings - - - - - - - - - - - - - - - - - - - - - - - -
    -
    - -
    -

    5 - Feature engineering: dummies and embeddings

    -

    Getting More Out of Feature Engineering and Tuning for Machine Learning

    - -
    -
    - -
    -
    -

    Getting set up

    -
    -
    library(tidymodels)
-library(embed)
-library(extrasteps)
-
-tidymodels_prefer()
-theme_set(theme_bw())
-options(pillar.advice = FALSE, pillar.min_title_chars = Inf)
-
-# Load our example data for this section
-"https://github.com/tidymodels/workshops/raw/refs/heads/2025-GMOFETML/slides/leaf_data.RData" |> 
-  url() |> 
-  load()
    -
    -
    -
    -
    -

    Leaf data

    - -
    -
    -

    Leaf data set

    -

    Slightly modified version of modeldata::leaf_id_flavia.

    -
    -
    glimpse(leaf_data)
-#> Rows: 1,907
-#> Columns: 55
-#> $ species                    <fct> chinese_redbud, chinese_redbud, chinese_red…
-#> $ apex                       <fct> none, none, none, none, none, none, none, n…
-#> $ base                       <fct> none, none, none, none, none, none, none, n…
-#> $ shape                      <fct> heart_shape, heart_shape, heart_shape, hear…
-#> $ edge                       <chr> "smooth", "smooth", "smooth", "smooth", "sm…
-#> $ outlying_polar             <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
-#> $ skewed_polar               <dbl> 0.4610066, 0.4747877, 0.5356883, 0.4771338,…
-#> $ clumpy_polar               <dbl> 0.006479784, 0.011100482, 0.017317486, 0.01…
-#> $ sparse_polar               <dbl> 0.01449941, 0.01451566, 0.02953578, 0.01425…
-#> $ striated_polar             <dbl> 0.9788360, 0.9797980, 0.6758621, 0.9896373,…
-#> $ convex_polar               <dbl> 0.000899987, 0.000152532, 0.035230741, 0.00…
-#> $ skinny_polar               <dbl> 0.1177954, 0.5440493, 0.7764908, 0.7067394,…
-#> $ stringy_polar              <dbl> 1.0000000, 1.0000000, 0.8544140, 1.0000000,…
-#> $ monotonic_polar            <dbl> 0.026807610, 0.005554220, 0.068481538, 0.09…
-#> $ outlying_contour           <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
-#> $ skewed_contour             <dbl> 0.4954035, 0.5321818, 0.5451171, 0.4675013,…
-#> $ clumpy_contour             <dbl> 0.007898114, 0.010846498, 0.020271493, 0.01…
-#> $ sparse_contour             <dbl> 0.01449941, 0.01436114, 0.02903878, 0.01451…
-#> $ striated_contour           <dbl> 0.9744898, 0.9811321, 0.6766917, 0.9784946,…
-#> $ convex_contour             <dbl> 0.000479470, 0.000000000, 0.035908221, 0.00…
-#> $ skinny_contour             <dbl> 0.1810093, 1.0000000, 0.7806988, 0.1348225,…
-#> $ stringy_contour            <dbl> 1.0000000, 1.0000000, 0.8864787, 1.0000000,…
-#> $ monotonic_contour          <dbl> 0.000816260, 0.000008740, 0.001253195, 0.32…
-#> $ num_max_points             <dbl> 1, 3, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 6, 1…
-#> $ num_min_points             <dbl> 2, 4, 7, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 8, 2…
-#> $ diameter                   <dbl> 1255.830, 1227.826, 1113.228, 1219.185, 118…
-#> $ area                       <dbl> 936626.5, 917300.5, 855376.5, 901303.0, 885…
-#> $ perimeter                  <dbl> 3830.534, 3781.445, 3695.646, 3678.409, 383…
-#> $ physiological_length       <dbl> 1253, 1218, 1098, 1205, 1153, 1179, 1188, 1…
-#> $ physiological_width        <dbl> 1088, 1091, 1083, 1053, 1078, 1073, 1054, 1…
-#> $ aspect_ratio               <dbl> 0.8683160, 0.8957307, 0.9863388, 0.8738589,…
-#> $ rectangularity             <dbl> 0.6870470, 0.6903027, 0.7193273, 0.7103222,…
-#> $ circularity                <dbl> 0.8021540, 0.8061315, 0.7870211, 0.8370680,…
-#> $ compactness                <dbl> 15.66578, 15.58849, 15.96701, 15.01237, 16.…
-#> $ narrow_factor              <dbl> 1.154255, 1.125413, 1.027912, 1.157821, 1.1…
-#> $ perimeter_ratio_diameter   <dbl> 3.050202, 3.079790, 3.319756, 3.017104, 3.2…
-#> $ perimeter_ratio_length     <dbl> 3.520711, 3.466036, 3.412416, 3.493266, 3.5…
-#> $ perimeter_ratio_lw         <dbl> 1.636281, 1.637698, 1.694473, 1.629056, 1.7…
-#> $ num_convex_points          <dbl> 125, 128, 114, 138, 125, 125, 121, 112, 115…
-#> $ perimeter_convexity        <dbl> 0.9404967, 0.9393550, 0.9232549, 0.9502414,…
-#> $ area_convexity             <dbl> 0.02359372, 0.03175513, 0.03932713, 0.01740…
-#> $ area_ratio_convexity       <dbl> 0.9769501, 0.9692222, 0.9621610, 0.9828892,…
-#> $ equivalent_diameter        <dbl> 1092.0393, 1080.7142, 1043.5992, 1071.2491,…
-#> $ eccentricity               <dbl> 0.44400376, 0.26866788, 0.31952502, 0.44811…
-#> $ contrast                   <dbl> 38.69163, 32.73154, 23.20849, 30.14546, 23.…
-#> $ correlation_texture        <dbl> 0.9959609, 0.9967740, 0.9975354, 0.9968981,…
-#> $ inverse_difference_moments <dbl> 0.5951133, 0.6022763, 0.6363508, 0.6135506,…
-#> $ entropy                    <dbl> 6.291552, 6.273267, 5.764995, 6.203471, 5.9…
-#> $ mean_red_val               <dbl> 38.02429, 34.55425, 30.92509, 33.44151, 33.…
-#> $ mean_green_val             <dbl> 72.38449, 69.89448, 69.22250, 71.32743, 70.…
-#> $ mean_blue_val              <dbl> 42.78902, 42.93044, 40.80089, 40.73282, 42.…
-#> $ std_red_val                <dbl> 41.66486, 40.48649, 38.55110, 39.09408, 40.…
-#> $ std_green_val              <dbl> 74.26665, 73.05275, 76.77596, 75.85529, 75.…
-#> $ std_blue_val               <dbl> 46.36232, 48.04195, 48.09177, 46.57170, 48.…
-#> $ correlation                <dbl> -0.027629688, -0.003103965, -0.036341630, 0…
    -
    -
    -
    -

    Leaf data set

    - -
    -

    Lakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077.

    -
    -
    -
    -

    Leaf data set

    - -
    -

    Lakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077.

    -
    -
    -
    -

    Leaf data set

    - -
    -

    Lakshika, J. P., & Talagala, T. S. (2021). Computer-aided interpretable features for leaf image classification. arXiv preprint arXiv:2106.08077.

    -
    -
    -
    -

    Your turn

    -

    -

    Load and explore the leaf data

    -

    Make time to look at the edge column and think about how one would encode it into numerics

    -
    -
    -
    -
    -05:00 -
    -
    -
    -
    -
    -

    Leaf edges

    -
    -
    -
      -
    • Low number of unique levels
      • -
    • Some overlap
      • -
    -
    -
    -
    leaf_data |>
-  count(edge)
-#> # A tibble: 7 × 2
-#>   edge                 n
-#>   <chr>            <int>
-#> 1 ""                  62
-#> 2 "denate"            65
-#> 3 "lobed"            109
-#> 4 "lobed, smooth"     52
-#> 5 "lobed, toothed"   109
-#> 6 "smooth"          1204
-#> 7 "toothed"          306
    -
    -
    -
    -
    -
    -

    Advanced Dummies

    - -
    -
    -

    Leaf edges dummies

    -

    (technically one-hot encoding)

    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Xdenatelobedlobed..smoothlobed..toothedsmoothtoothed
    0000100
    0000001
    0000010
    0000010
    0000001
    0010000
    -
    -
    -
    -
    -

    Dummy variables

    -
    -
    -

    Pros

    -
      -
    • Commonly used
      • -
    • Easy interpretation
      • -
    • Will rarely lead to a decrease in performance
      • -
    -
    -

    Cons

    -
      -
    • Can create many columns
      • -
    • Needs clean levels
      • -
    • Not likely the most efficient representation
      • -
    -
    -
    -
    -

    Overlapping

    -
    -
    -

    Some of the labels

    -
      -
    • lobed
      • -
    • smooth
      • -
    • toothed
      • -
    • lobed, smooth
      • -
    • lobed, toothed
      • -
    -
    -

    If you want to find a “toothed” leaf, which level do you pick?

    -
      -
    • toothed
      • -
    • lobed, toothed
      • -
    • toothed and lobed, toothed
      • -
    -
    -

    We can let lobed, toothed be counted for lobed and toothed

    -
    -
    -
    -
    -

    Advanced Dummies

    -

    This is basically a poor man’s Natural Language Processing.

    -


    -
    -

    tokenization -> counting

    -
    -


    -

    We think it is a frequent enough case that it is considered its own method.

    -

    We have 2 variants: “extraction” and “multi choice”

    -
    -
    -

    Advanced Dummies - Extraction

    -

    Works on a singular column, using a regular expression to extract the items we want to count.

    -

    Done using regular expressions, either by specifying sep to split the string by, or by using pattern to extract the items.

    -

    Allows for 0 to many items in each string.

    -

    Implemented as step_dummy_extract().

    -
    -

    FEAZ

    -
    -
    -
    -

    Advanced Dummies - Extraction

    -
    -
    -

    Input

    -
    -
    ""
-"denate"
-"lobed"
-"lobed, smooth"
-"lobed, toothed"
-"smooth"
-"toothed"
    -
    -


    -

    Using

    -

    sep = ", "

    -
    -

    Result

    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    denatelobedsmoothtoothed
    0000
    1000
    0100
    0110
    0101
    0010
    0001
    -
    -
    -
    -
    -
    -

    Advanced Dummies - Extraction

    -
    -
    -

    Input

    -
    -
    "Etching on paper"
-"Oil paint on canvas"
-"Acrylic paint on paper"
-"Oil paint and wax on canvas"
-"Oil paint, ink on canvas"
    -
    -


    -

    Using

    -

    sep = ", "

    -
    -

    Result

    -
    -
    - -------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Acrylic.paint.on.paperEtching.on.paperink.on.canvasOil.paintOil.paint.and.wax.on.canvasOil.paint.on.canvas
    010000
    000001
    100000
    000010
    001100
    -
    -
    -
    -
    -
    -

    Advanced Dummies - Extraction

    -
    -
    -

    Input

    -
    -
    "Etching on paper"
-"Oil paint on canvas"
-"Acrylic paint on paper"
-"Oil paint and wax on canvas"
-"Oil paint, ink on canvas"
    -
    -


    -

    Using

    -

    sep = "(, )|( and )|( on )"

    -
    -

    Result

    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Acrylic.paintcanvasEtchinginkOil.paintpaperwax
    0010010
    0100100
    1000010
    0100101
    0101100
    -
    -
    -
    -
    -
    -

    Advanced Dummies - Extraction

    -
    -
    -

    Input

    -
    -
    "['red', 'blue']"
-"['red', 'blue', 'white']"
-"['blue', 'blue', 'blue']"
    -
    -


    -

    Using

    -

    pattern = "(?<=')[^',]+(?=')"

    -
    -

    Result

    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - -
    blueredwhite
    110
    111
    300
    -
    -
    -
    -
    -
    -

    Advanced Dummies - Multi Choice

    -

    Works on multiple columns, counting items across columns.

    -

    It can be seen as joining multiple applications of dummy variables together.

    -

    Allows for 0 to many items.

    -

    Implemented as step_dummy_multi_choice().

    -
    -

    FEAZ

    -
    -
    -
    -

    Advanced Dummies - Multi Choice

    -
    -
    -

    Input

    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    lang_1lang_2lang_3
    EnglishNAItalian
    SpanishFrenchFrench
    NANANA
    EnglishNANA
    -
    -
    -
    -

    Result

    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    EnglishFrenchItalianSpanish
    1010
    0101
    0000
    1000
    -
    -
    -
    -
    -
    -

    Othering

    -

    Both step_dummy_extract() and step_dummy_multi_choice() contain a threshold argument.

    -

    This is used to combine infrequent levels together.

    -

    We have a step step_other() that does this for nominal variables, but it doesn’t work in these cases, as it has to happen after the extraction/combination.

    -
    -
    -

    Othering

    -

    threshold = 0 produces 1217 columns.

    -
    -
    - ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    X1.monitorX1.personX1.projectionX10.light.boxesX10.tranformersX100.digital.printsX100.worksX11.photographsX11.worksX12.ceramic.tea.mugsX12.drawingsX13.canvasesX133.slidesX14.monitorsX14.photographsX14.small.cubesX14.worksX15.engraved.plaquesX15.hand.coloured.photographsX15.photographsX15.roomsX15.steel.kitchen.ustensilsX15.unglazed.ceramic.formsX151.text.panelsX16.channelsX16.mmX16.paintingsX16.sheetsX160.slidesX16mmX18.drawingsX18.photographsX19.wooden.pieces.of.furnitureX2.aluminium.panelsX2.aluminium.tablesX2.back.projectionsX2.bannersX2.canvasesX2.car.batteriesX2.copper.panelsX2.digital.printsX2.electric.cablesX2.flat.screensX2.fluorescent.tubesX2.hooksX2.horsesX2.light.bulbsX2.lithographsX2.mapsX2.marker.pensX2.metal.scaffold.towersX2.monitorsX2.paintingsX2.papersX2.peopleX2.photographsX2.porcelain.light.socketsX2.projectionsX2.ropesX2.screenprintsX2.sections.of.pre.cast.concrete.pipeX2.sheetsX2.slippersX2.stainless.steel.discsX2.trainsX2.transparenciesX2.videosX2.wall.piecesX2.worksX20.cotton.mattressesX20.flat.screens.or.1.projectionX20.photographsX20.wooden.bedsX2000.gerberasX201.photographsX21.aluminium.bricksX21.channelsX21.worksX22.photographsX220.worksX23.monitorsX24.light.bulbsX24.worksX25.photographsX26.worksX27.photographsX28.etchingsX3.ballsX3.booksX3.canvasesX3.flat.screensX3.Kentia.palms.in.3.terracotta.potsX3.lampsX3.light.bulbsX3.monitorsX3.peopleX3.photographsX3.printersX3.printsX3.projectionsX3.screenprintsX3.skateboardsX3.transformersX3.wall.mounted.LCD.monitorsX3.worksX30.light.emitting.diodes.unitsX30.worksX31.photographsX32.digital.printsX32.flat.screensX32.papersX320.slidesX34.photographsX34.wooden.sculpturesX35.mmX35.worksX350.digital.printsX355.photographsX35mmX36.tin.dogsX39.cardboard.boxesX39.metronomesX4.aluminium.pixel.boxes.with.DMX.control.boxX4.Belisha.beaconsX4.canvasesX4.channelsX4.corrugated.slabs.of.concreteX4.digital.printsX4.digital.prints.with.acrylic.paintX4.fibreboard.panelsX4.flat.screensX4.gascookersX4.hardboard.wallsX4.light.bulbsX4.mannequinsX4.monitorsX4.photocopiesX4.photographsX4.projectionsX4.steel.capsulesX40.light.emitting.diode.unitsX40.photographsX40.worksX42.electric.lampsX42.photographsX46.photographsX49.Perspex.boxesX5.digital.printsX5.drawingsX5.flat.screensX5.knivesX5.monitorsX5.painted.plaster.bustsX5.painted.wooden.shelvesX5.photographsX5.projectionsX5.tablesX5.worksX50.canvasesX50.cardboard.boxesX50.slidesX53.photographsX54.digital.printsX6.digital.printsX6.fabricsX6.lithographsX6.monitorsX6.Perspex.panelsX6.photographsX6.projectionsX6.worksX60.digital.printsX60.worksX62.florescent.lightsX64.tin.soldiersX7.channelsX7.drawingsX7.light.emitting.diode.columnsX7.panelsX7.projectionsX7.stoolsX7.worksX71.vinyl.recordsX76.worksX8.channelsX8.digital.printsX8.headphonesX8.letterpress.printsX8.mmX8.monitorsX8.projectionsX8.synthetic.fabric.flagsX8.woodblockX8.worksX80.slidesX800.digital.printsX84.sleevesX9.doorsX9.LED.lightsX9.photographsX9.worksX90.photographsacetateacrylicacrylic.fibreacrylic.glassacrylic.paiacrylic.paintAcrylic.paintacrylic.resinacrylic.sheetAcrylic.sheetacrylic.sheetsAcrylic.tubesadhesiveadhesive.tapeaerialsAgfa.Isolette.cameraAlabasteralarm.clocksAlkyd.paintAltered.Eames.plywood.leg.splintsaluminiumAluminiumaluminium.bucklesaluminium.discaluminium.lightboxAluminium.machinery.partaluminium.paintaluminium.panelaluminuimAluminuimaluminum.panelamplifierand.soundand.sound..monoand.sound..mono.and.sound..stereo.animal.bonesAnimal.intestinesaquatintAquatintaquatint.with.hand.colouringartificial.foliageartificial.mossartificial.sandartificial.wigartist.s.hairashAshashtrayaudioAudioaudio.systemaudiotrackback.projectionbadgebadges.with.printed.papersballastsbalsa.woodBamboo.caneBamboo.polesbarkbaseball.capbath.bombsbatteribauxite.rocksbeadsBeanbagbeechbeer.canbellsbicycle.beltbindisbinsblackblackboardBlind.embossed.printboardBoardBoatbonebonesbookBookbook.coverbookletsbooksbootsbowlboxbrassBrassbrass.pinsbrass.platebreeze.blocksbrickBriefcasebronzeBronzeBronze.bellsbronze.powderBronze.with.silk.scarfBronze..with.Jamie.SargeantBronze..with.Nicholas.Sloanbroomsbucketsbuckrambuoyburlaq.sackburnt.woodButterfliesbuttonscablecablescalfcalicocandlecanvasCanvasCanvas.lining.with.ingrained.dustcanvas.paperCanvas.tacking.edgescarbon.fibrecarborundumcarborundum.mezzotintcardCardcardboardCardboardcardboard.architectural.modelcardboard.basecardboard.boxCardboard.boxcardboard.boxescardboard.coffincarpetCarpetcarpetsCarrara.marbleCast.ironcastor.wheelsCedar.woodceilingCellophanecellulosecellulose.lacquercellulose.printcementcement.blockscentral.processing.unitceramicCeramicceramic.tileschainchairchair.partschairschalkChalkcharcoalCharcoalchine.colléchipboardChipboardchromechrome.steel.ballsChromogenic.printcibachromecibachrome.printCibachrome.printcigarette.ashcigarette.boxescigarette.buttscigarette.sheetscigarettesclayClayclothesclovescoalcoatCoca.cola.bottlecoconut.oilcoffeecoincollagecollagraphcolourcolour.andcolour.ans.sound..stereo.coloured.graphitecoloured.light.bulbscoloured.pencilcoloured.pencilscolumnsCommercial.paintcompressorcompressorscomputerConcreteContéContractor.s.shedCooked.couscouscookercoppercopper.sulphatecopperplatecopperplatedcoralcordCork.panelscorrection.fluidCorrection.fluidcottonCottoncotton.costumecotton.threadcotton.woolcowCowhidecrayonCrayoncrude.oilcrystalline.particlescue.standcuescurtainsCut.paperDelabole.slatedesksdetritusdigging.toolsdigital.imagedigital.printDigital.printDigital.print.with.acrylic.paintdigital.printsDigital.prints.with.acrylic.paintdollsdoordrawingsDresden.D1.projectordress.shirtdry.pigment.paintdrypointDrypointduck.tapedyeearthEarthearthenwareEarthenwareelectric.cableelectric.wireelectrical.aelectrical.cableelectrical.componentselectrical.pumpelectronic.circuitelephant.dungemulsionEmulsionenamelEnamelenamel.buttonenamel.paintEnamel.paintencre.de.ChineEngineengravingEngravingenvelopesepoxyetchingEtchingEtching.with.hand.colouringexpanding.foameyebrow.pencileyeshadowfabricFabricfabric.patchesfaced.particleboardfan.heaterFeathersFed.2.type.camerafeltFeltfelt.penFerric.oxidefibre.clayfibreboardFibreboardFibreboard.cabinetfibreboard.panelsfibreboard.plinthfibreboard.with.clayfibreglassFibreglassfilmFilmFilm.16.mmflat.screenflat.screen.or.projectorflaxFliesFlintfloodlightflowersfluorescent.lightfluorescent.lightsfluorescent.site.jacketfluorescent.tubesflyersfoamfoam.corefoam.rubberfoilfoil.tapefoodformaldehyde.solutionformicaFormicafound.postersframed.Futon.mattressGallery.lightinggalvanised.steelgalvanized.steelgelgelatin.silver.printgelatin.silver.printon.papergelatine.silver.print.with.dyeglasglassGlassglass.aglass.beadsglass.boxGlass.chandelierglass.vitrine.containing.beauty.andGlass.10.hardbacked.booksglitterglovegluegobogold.leafgold.metallic.paintGold.paintgouacheGouacheGranitegraphiteGraphitegrillGrpahiteguitar.stringsgunGunpowderhairhair.over.metalHand.coloured.photographhand.colouringHand.cut.bookhandcolouringhardboardhardboard.plinthhardboard.tablehardwoodhatheadphonesHeart.to.HearHeavy.goods.vehiclehelium.balloonhemp.cordhessianHessianhigh.definitionhigh.definition_2hookhornhosepipesHousehold.emulsion.painthousehold.gloss.painthousehold.paintHousehold.painthuman.hairhydraulic.ramsidentity.cardinfrared.sensorsingrained.dustinkInkinteractiveIris.printironiron.powderjacketJacketjutekapokkey.Kilkenny.limestonekitchen.utensilskiteknifellab.coatlacquerLacquered.woodladderlambLambda.printlampsLampslatexleadLeadleatherLeatherleather.briefcaseletterpressLetterpresslicelightlight.boxlight.boxeslight.bulblight.bulbslight.control.unitlightboxLightboxeslighterlighting.systemlightslinenlinocutLinocutLinocut.printlinoleumlithographLithographlithograph.with.plasticineLithograph.woodcutlithographsmagnetic.rodsmagnetsMahoganymango.seedsmannequinMannequinmapMapmap.pinsmarbleMarblemarble.basemarker.penMarker.penMDFMDF.backboardmemelaminemelanine.fibreboardMelinexmetalMetalMetal.bicyclemetal.bottle.capsmetal.bucketmetal.chainmetal.clampmetal.clothes.railMetal.cotmetal.detritusmetal.film.spoolmetal.foilMetal.frameMetal.framesmetal.hookmetal.pipemetal.sheetmetal.stringMetal.tableMetal.tin.cansmetal.wiremetalcut.printsmetallic.powdermezzotintMezzotintmica.flakesmicrophonemin.DV.cam.tapemini.disc.playermirrorMirrormirrorsMiss.Piggy.Bag..and.contents.Mixed.mediamobile.phomodelling.puttymonitormonitor.or.flat.screenmonitor.or.projectionmonoMonoprintmonotypeMonotypemorse.code.unitmotormotorised.basemounMountain.bicycleMountain.peak.in.containermountaineering.equipmentmountedmounted.onto.aluminiumMudmultiple.projectionsmuslinMylar.screennailsnatural.fibresneon.gasneon.lightsNeon.lightsneon.paintnewspapersnotebooknylonnylon.strapsnylon.stringsNylon.tightsnyloprintsoakOakoak.tableoak.twigofficeoiloil.paintOil.paintoil.pasteloil.stickOil.stickoil.tinton.47.panelson.aluminiumon.aluminium.panelon.boardon.cardboardon.paperon.paper.between.glasson.paper.mountedon.paper.mounted.onto.acrylic.glasson.paper.mounted.onto.aluminiumon.paper.mounted.onto.aluminium.panelon.paper.mounted.onto.aluminuimon.paper.mounted.onto.boardon.paper.mounted.onto.panelon.paper.mounted.onto.paperon.paper.mounted.onto.Perspexon.paper.mounted.onto.plasticon.paper.with.chalkon.paper.with.dry.transfert.print.mounted.onto.paperon.paper.with.painton.paperson.papers.with.painton.plasticon.sticker.paperon.vinyl.mounted.onto.aluminiumopen.cell.foamoptical.gelatin.silver.fibre.printor.videoOrganOstrich.eggother.mother.materiaother.materialsothers.materialsoxidised.brassppaintPaintPaint.brushpainted.aluminiumPainted.aluminiumpainted.boardsPainted.fibreboardpainted.glasspainted.MDFPainted.steelpainted.wallpainted.wall.textpainted.woodPainted.woodPainted.wooden.building.with.asphalt.shingle.roofpalm.frondsPalm.treepanelpaperPaperpaper.mountedpaper.mounted.onto.aluminiumpaper.mounted.onto.aluminium.panelpaper.mounted.onto.boardpaper.mounted.onto.foam.corepaper.mounted.onto.muslinpaper.mounted.onto.panelpaper.mounted.onto.paperpaper.pulppaper.tapepaper.with.dry.transfer.printpaper.with.dyepaper.with.inkpaper.with.oil.paintpaper.with.watercolourpaper..Verso..inkpapersparaffinparaffin.lamppassportpastelPastelpatinapeacock.featherspenpendant.lampspeoplePerformanceperfume.flask.with.pouchperspexPerspexPewterpharmaceutical.packagingphoto.etchingPhoto.etchingphotographPhotographphotograph.mountedPhotographic.contact.sheetphotographsPhotographsphotographs.PianopigmentPigmentPigment.transferpigmentsPine.plywoodpinsplantsplasterPlasterplaster.figuresplaster.powderplasterboardplasticPlasticplastic.bagplastic.bagsplastic.beadsPlastic.beadsplastic.boxplastic.boxesplastic.coverPlastic.lidsplastic.pearlsplastic.pen.lidsplastic.pillowplastic.shoeplastic.threadplastic.tubesplastic.watch.with.photographplastic.watering.canplastic.watering.can..spray.botplasticinePlasticineplatinium.printPlexiglasplexiglassplinthplug.boardsplywoodPlywoodplywood.boardpoPolychromed.aluminiumPolyesterpolyester.foampolyester.resinPolyester.resinpolyester.textilepolyfibrepolymerpolystyrenepolystyrene.foampolythenepolyurethanepolyurethane.foamPolyurethane.resinPolyurethane.rubberpolyvinyl.acetate.paintPolyvinyl.acetate.paintporcelainPorcelainporcelain.modelporcelaineportable.keyboard.keysPortorino.marblepostcardPostcardpostcardspostersPotato.printPowder.coated.aluminiumpowder.coated.steelPowder.coated.steelpowder.paintpowder.coated.steel_2Powder.coated.steel_2printPrintprinted.mapprinted.pprinted.paperPrinted.paperprinted.papersPrinted.papersprojectionprojection.or.7.monitorsprojection.or.monitorpumpsPVCradioradio.transmitterraminrecipesRecord.deckrecord.playerrecorded.voicereel.to.reel.tape.deckreliefReliefRelief.printresinResinresin.baseresin.blockribbonRibbonRiver.mudRoman.vesselroperopesrosary.beadsrubberRubberrubber.coated.steelRubber.inner.tubesrubber.paddingsafety.pinssaltsandsatinsatin.ribbonsawdustSax.oil.paintscanachrome.printScanachrome.printscissorsscreenprintScreenprintScreenprint.with.acrylic.varnishScreenprintsscrewssea.shellsseatsection.of.concrete.wallsellotapeSellotapeSequinssewing.machinesheep.excrementshellacshellac.resinshellsShellsshieldshirtshoeboxesshoelaceshoesShop.mannequinshown.asshown.as.videoshreddersilicon.hosesilicon.tubingsiliconesilicone.adhesivesilicone.rubberSilicone.rubbersilk.tiesilkcreen.printsilkscreenSilkscreenSilkscreen.printsilversilver.foilsilver.gelatin.printsilver.solderSilver.teapotSilverpointsisal.ropesisal.stringSlateslideSlideslidessmall.speakerssmokesmoke.machinesosoapSoapsocksSofasoftwareSoftwareSondor.playback.machinesoundsound..monosound..mono.sound..stereo.sound..surround.sound.recordingsound..woodspeakersspicesspinning.topspray.enamelspray.paintSpray.paintSprayed.Q.CellsreenprintStack.of.printed.paperstainless.steelStainless.steelstainless.steel.ashtraystainless.steel.screwsstainless.steel.tea.urnstainless.steel.teapotstainless.steel.wirestandstaple.gunStarfishsteelSteelsteel.barsSteel.bedspringssteel.bracketsteel.brake.wiressteel.cableSteel.lockerSteel.platesteel.screwssteel.shelvingsteel.wirestencilstereostereo.stickersstockingsstone.basestone..with.Jamie.Sargeantstoolstoryboardsstrawstrinstringstyrenestyrofoamsugar.papersuitcaseSuper.16.mmSuper.8Super.8.mmsweet.wrappersswing.seatsynthetic.fibresynthetic.fibrestableTabletablestapetartarpaulintaxidermyteatelephoneTemperatenttextileTextiletextilesTextilestheatre.lightthreadthreadsticketstightsTimbertissuetoilettooth.btorchlighttoy.eyeballstracing.papertracking.systemtram.windowtransfer.letteringtransparencyTransparencytransparent.papertree.branchestripodtrouserstwigstwo.pack.car.lacquertypetypescripttypewritten.inkUHF.Radioumbrellaunderpantsvarnishvelvetvelvet.pVHS.tapevideoVideovideo.cameraviewfindervinylVinyl.dispersionvinyl.floor.coveringvinyl.paintVinyl.recordvinyl.seatVinyl.tapevinyl.textvinyl.wall.textVinyl.wall.textvinyl.wall.textswalkmanWalkmanwallwall.clockWall.paintingwall.textWallpaperWasherswaterwatercolourWatercolourwaxweb.search.programwebbingWestern.red.cedarwheelswhitewirewire.coat.hanger.wireswoodWoodWood.engravingwood.traywood.varnishwood.veneerWoodblockwoodcutWoodcutWooden.abacus.beadswooden.basewooden.beamsWooden.billiard.tableWooden.birdhouse.with.metal.roofwooden.boxesWooden.cabinetwooden.ceiling.propswooden.chairWooden.chairWooden.constructionWooden.deskwooden.dowelswooden.floorwooden.frameswooden.glass.negative.plate.carrierwooden.palletswooden.panelwooden.plankswooden.platformwooden.stepswooden.tableWooden.tablewooden.trestlewoolWoolwork.overallsworksWorry.beadswrapszincother
    00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000
    00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
    00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
    00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
    00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
    00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
    -
    -
    -
    -
    -

    Othering

    -

    threshold = 0.05 produces 11 columns.

    -
    -
    - ------------- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    aquatintcanvascolourEtchinggelatin.silver.printLithographon.paperpaperPhotographScreenprintother
    00100000003
    00010001000
    00010001000
    00010001000
    01000000001
    01000000001
    -
    -
    -
    -
    -

    Your turn

    -

    -

    Figure out whether step_dummy_extract() or step_dummy_multi_choice() is most appropriate on the edge variable in leaf_data and apply it

    -
    -
    -
    -
    -03:00 -
    -
    -
    -
    -
    -
    -

    Dimensionality Reduction

    - -
    -
    -

    Dimensionality Reduction

    -

    What is it?

    -

    Techniques that remove or alter features in order to have fewer but more informative features.

    -

    Why do we do it?

    -
      -
    • redundant information
      • -
    • ineffective representation
      • -
    • computational speed
      • -
    -
    -
    -

    Redundant Information

    -

    A feature with no information could be included.

    -
    -
    leaf_data |>
-  count(outlying_contour)
-#> # A tibble: 1 × 2
-#>   outlying_contour     n
-#>              <dbl> <int>
-#> 1                0  1907
    -
    -
    -
    -

    Redundant Information

    -

    Or duplicated or functionally equivalent features

    -
    -
    leaf_data |>
-  ggplot(aes(area, equivalent_diameter)) +
-  geom_point()
    - -
    -

    -Figure 1 -

    -
    -

    Ineffective Representation

    -

    While keeping our models in mind, we want to make sure the data is well-suited

    -
      -
    • Correlated data -
        -
      • hard for some models
        • -
      • -
    • lat/lon compared to distance/angle -
        -
      • hard for most models
        • -
      • -
    -
    -
    -

    Ineffective Representation

    - -

    -Figure 2 -

    -
    -

    Ineffective Representation

    - -

    -Figure 3 -

    -
    -

    Computational Speed

    -

    Depending on what method we are using and how the data is affected by it, we could see a large reduction in features. This, in turn, leads to a smaller model that is faster to train on.

    -

    Only exploration and trial and error can determine whether you should use dimensionality reduction techniques. Knowing which methods do what helps you determine what to try.

    -
    -
    -

    Dimensionality Reduction Method

    -
      -
    • Zero Variance removal
      • -
    • PCA -
        -
      • Truncated PCA
        • -
      • Sparse PCA
        • -
      • -
    • NNMF
      • -
    • UMAP
      • -
    • Isomap
      • -
    -
    -
    -

    Restrictions

    -

    All the methods shown today will not be able to handle

    -
      -
    • missing data
      • -
    • Non-numeric data
      • -
    -
    -
    -

    Why not t-SNE?

    -

    One of the main requirements for a feature engineering method is that you can reapply the trained transformation done on the training data set to the testing data set.

    -

    This is not possible with t-SNE as it is an iterative method that shifts observations in the lower-dimensional space based on their distances to points in the higher-dimensional space.

    -

    It doesn’t create a mapping that can be reused.

    -
    -

    Visualizing Data using t-SNE, Laurens van der Maaten and Geoffrey Hinton, Journal of Machine Learning Research, 2008.

    -
    -
    -
    -

    PCA

    -

    Principal Compoment Analysis is a linear combination of the original data such that most of the variation is captured in the first variable, then the second, then the third, and so on.

    -
    -

    FEAZ, FES

    -
    -
    -
    -

    PCA Algorithm

    -

    The first principal component of a set of features \(X_1, X_2, ..., X_p\) is the normalized linear combination of the features.

    -

    \[ -Z_1 = \phi_{11} X_1 + \phi_{21} X_2 + ... + \phi_{p1}X_p -\]

    -

    that has the largest variance under the constraint that \(\sum_{j=1}^p \phi_{j1}^2 = 1\).

    -

    we refer to \(\phi_{11}, ..., \phi_{p1}\) as the loadings of the first principal component.

    -

    And think of them as the loading vector \(\phi_1\).

    -
    -
    -

    PCA Algorithm

    -

    since we have \(z_{i1} = \phi_{11} x_{i1} + \phi_{21} x_{i2} + ... + \phi_{p1}x_{ip}\), then we can write

    -

    \[ -\underset{\phi_{11}, ..., \phi_{p1}}{\text{maximize}} \left\{ \dfrac{1}{n} \sum^n_{i=j} z_{i1} ^2 \right\} \quad \text{subject to} \quad \sum_{j=1}^p \phi_{j1}^2 = 1 -\]

    -

    We are, in essence, maximizing the sample variance of the \(n\) values of \(z_{i1}\).

    -

    We refer to \(z_{11}, ..., z_{n1}\) as the scores of the first principal component.

    -
    -
    -

    PCA Algorithm

    -

    Luckily, this can be solved using techniques from Linear Algebra, more specifically, it can be solved using an eigen decomposition.

    -

    One of the main strengths of PCA is that you don’t need to use optimization to get the results without approximations.

    -
    -
    -

    PCA Algorithm

    -

    Once the first principal component is calculated, we can calculate the second principal component.

    -

    We find the second principal component \(Z_2\) as a linear combination of \(X_1, ..., X_p\) that has the maximal variance out of the linear combinations that are uncorrelated with \(Z_1\)

    -

    this is the same as saying that \(\phi_2\) should be orthogonal to the direction \(\phi_1\)

    -
    -
    -

    How is that a dimensionality reduction method?

    -

    By itself, it isn’t, as it rotates all the features in the feature space.

    -

    It becomes a dimensionality reduction method if we only calculate some of the principal components.

    -

    This is typically done by retaining a specific number of components or as a threshold on the variance explained.

    -
    -
    -

    4 different percent variance plots

    - -Faceted bar chart. 4 bar charts in a 2 by 2 grid. Each one represents a different data set which has had PCA applied to it.
    -
    -

    Applying PCA with recipes

    -

    Either use the num_comp argument.

    -
    -
    rec <- recipe(mpg ~ ., data = mtcars) |>
-  step_normalize(all_predictors()) |>
-  step_pca(all_numeric_predictors(), num_comp = 5)
    -
    -


    -

    or using the threshold argument

    -
    -
    rec <- recipe(mpg ~ ., data = mtcars) |>
-  step_normalize(all_predictors()) |>
-  step_pca(all_numeric_predictors(), threshold = 0.8)
    -
    -
    -
    -

    Your turn

    -

    -

    Apply PCA using step_pca() to leaf_data data set.

    -

    Experiment with different values of num_comp and or threshold.

    -
    -
    -
    -
    -05:00 -
    -
    -
    -
    -
    -

    PCA Pros and Cons

    -
    -
    -

    Pros

    -
      -
    • Fast
      • -
    • Reliable
      • -
    • Exact results (up to sign changes)
      • -
    -
    -

    Cons

    -
      -
    • Computational time is linear in the number of columns
      • -
    • Can be quite hard to interpret
      • -
    -
    -
    -
    -

    Truncated PCA

    -
    -

    Computational time is linear in the number of columns

    -
    -

    By default, step_pca() calculates all the loading vectors. And then subset them down to what you need.

    -

    Instead, we can use a different implementation that only calculates what you need. This is what we call truncated PCA.

    -
    -

    FEAZ

    -
    -
    -
    -

    Truncated PCA with recipes

    -

    Can only be done using num_comp

    -
    -
    library(embed)
-
-rec <- recipe(mpg ~ ., data = mtcars) |>
-  step_normalize(all_predictors()) |>
-  step_pca_truncated(all_numeric_predictors(), num_comp = 5)
    -
    -
    -
    -

    Sparse PCA

    -
    -

    Can be quite hard to interpret

    -
    -

    Every component is a linear combination of all predictors.

    -

    \[ -\begin{alignat}{4} -PC1 &= cyl \cdot -0.021 &&+ disp \cdot -0.85 &&+ hp \cdot -0.52\\ -PC2 &= cyl \cdot 0.013 &&+ disp \cdot -0.52 &&+ hp \cdot 0.85\\ -PC3 &= cyl \cdot -0.12 &&+ disp \cdot 0.016 &&+ hp \cdot 0.081\\ -PC4 &= cyl \cdot -0.22 &&+ disp \cdot -0.0061 &&+ hp \cdot 0.033\\ -PC5 &= cyl \cdot 0.73 &&+ disp \cdot -0.014 &&+ hp \cdot 0.0016 -\end{alignat} -\]

    -

    If we could force some of the loadings to be 0, it would reduce things a lot.

    -
    -

    FEAZ

    -
    -
    -
    -

    Sparse PCA with recipes

    -

    Can only be done using num_comp.

    -

    The predictor_prop argument is used to determine how many zeroes in the loadings.

    -
    -
    library(embed)
-
-rec <- recipe(mpg ~ ., data = mtcars) |>
-  step_normalize(all_predictors()) |>
-  step_pca_sparse(all_numeric_predictors(), num_comp = 5, predictor_prop = 0.8)
    -
    -
    -
    -

    Sparse PCA with recipes

    -

    predictor_prop = 0.8

    -
    -
    recipe(mpg ~ ., data = mtcars) |>
-  step_normalize(all_predictors()) |>
-  step_pca_sparse(all_numeric_predictors(), num_comp = 4, predictor_prop = 0.8) |>
-  prep() |>
-  tidy(number = 2) |>
-  pivot_wider(names_from = component, values_from = value) |>
-  select(-id)
-#> # A tibble: 10 × 5
-#>    terms      PC1    PC2     PC3      PC4
-#>    <chr>    <dbl>  <dbl>   <dbl>    <dbl>
-#>  1 cyl   -0.503    0      0.0755  0      
-#>  2 disp  -0.504    0      0       0.227  
-#>  3 hp    -0.377   -0.228 -0.0985  0      
-#>  4 drat   0.267   -0.262  0       0.911  
-#>  5 wt    -0.420    0.105 -0.383   0.236  
-#>  6 qsec   0        0.473 -0.425   0.0966 
-#>  7 vs     0.313    0.203 -0.429  -0.0774 
-#>  8 am     0.0687  -0.440  0.191  -0.00802
-#>  9 gear   0       -0.482 -0.237  -0.208  
-#> 10 carb  -0.00616 -0.422 -0.617  -0.0654
    -
    -
    -
    -

    Sparse PCA with recipes

    -

    predictor_prop = 0.2

    -
    -
    recipe(mpg ~ ., data = mtcars) |>
-  step_normalize(all_predictors()) |>
-  step_pca_sparse(all_numeric_predictors(), num_comp = 4, predictor_prop = 0.2) |>
-  prep() |>
-  tidy(number = 2) |>
-  pivot_wider(names_from = component, values_from = value) |>
-  select(-id)
-#> # A tibble: 10 × 5
-#>    terms   PC1    PC2    PC3   PC4
-#>    <chr> <dbl>  <dbl>  <dbl> <dbl>
-#>  1 cyl   0.712  0      0     0    
-#>  2 disp  0.702  0      0     0    
-#>  3 hp    0      0      0     0    
-#>  4 drat  0      0      0     0.978
-#>  5 wt    0      0     -0.278 0    
-#>  6 qsec  0      0.666  0     0    
-#>  7 vs    0      0      0     0    
-#>  8 am    0      0      0     0.211
-#>  9 gear  0     -0.746  0     0    
-#> 10 carb  0      0     -0.961 0
    -
    -
    -
    -

    NNMF

    -

    Non-Negative Matrix Factorization is conceptually similar to PCA, but it has different objectives.

    -

    PCA aims to generate uncorrelated components that maximize the variances. One component at a time.

    -

    NNMF, on the other hand, simultaneously optimizes all the components under the constraint that all the loadings are non-negative. While the data is also non-negative.

    -
    -

    FEAZ FES

    -
    -
    -
    -

    NNMF restriction

    -

    The data has to be non-negative, i.e., 0 or higher.

    -

    This might feel like a pretty big restriction. And that is not wrong. But a lot of data sets end up being naturally non-negative.

    -

    Or could at least be turned into non-negative ones with transformations. As long as you don’t scale them below 0.

    -

    It makes for much easier interpretations as the loadings don’t cancel each other out like they do for PCA.

    -
    -
    -

    NNMF Pros and Cons

    -
    -
    -
      -
    • More interpretable results
      • -
    • Pulls out better structures
      • -
    -
    -
      -
    • Data must be non-negative
      • -
    • Computationally expensive
      • -
    • Training depends on the seed
      • -
    -
    -
    -
    -

    NNMF with recipes

    -
    -
    set.seed(1234)
-
-recipe(mpg ~ ., data = mtcars) |>
-  step_nnmf_sparse(all_numeric_predictors(), num_comp = 2) |>
-  prep() |>
-  tidy(number = 1) |>
-  pivot_wider(names_from = component, values_from = value) |>
-  select(-id)
-#> # A tibble: 10 × 3
-#>    terms    NNMF1   NNMF2
-#>    <chr>    <dbl>   <dbl>
-#>  1 am    0        0.00631
-#>  2 carb  0.00428  0.0249 
-#>  3 cyl   0.0134   0.0177 
-#>  4 disp  0.639    0      
-#>  5 drat  0.00545  0.0188 
-#>  6 gear  0.00509  0.0240 
-#>  7 hp    0.294    0.840  
-#>  8 qsec  0.0316   0.0606 
-#>  9 vs    0.000293 0.00247
-#> 10 wt    0.00742  0.00519
    -
    -
    -
    -

    UMAP

    -

    Uniform Manifold Approximation and Projection is another method that takes high-dimensional data and transforms it into a lower-dimensional space.

    -

    Runs relatively fast and is popular in visualizations.

    -
    -
    -

    UMAP Algorithm

    -

    Rough algorithm

    -
      -
    1. Use spectral embedding to embed points in a low-dimensional space
      2. -
    2. Calculate similarity scores between points based on the original data set
      3. -
    3. Randomly samples a pair of points based on their similarity scores
      4. -
    4. Flips a coin to decide which of the pair of points to give to the other one
      5. -
    5. Randomly picks a non-neighbor point to move away from
      6. -
    6. Moves the selected point towards its neighbor and away from its non-neighbor
      7. -
    7. Repeat 3-6
      8. -
    -
    -
    -

    UMAP parameters

    -
      -
    • n_neighbors
      • -
    -

    Determines how many points are considered neighbors. A point counts as its own neighbor. Lower values lead to a local view.

    -
      -
    • min_dist
      • -
    -

    Determines how close points are allowed to be to each other in the low-dimensional space.

    -
      -
    • metric
      • -
    -

    How distances are calculated in the input data: euclidean, manhattan, jaccard, etc.

    -
    -
    -

    UMAP hesitancy

    -

    Due to the flexibility of how this method works, it is almost always possible to generate graphs that appear to have insights in them.

    -
    -
    -

    Which was created with random data?

    - -

    -Figure 4 -

    -
    -

    All of them! Different n_neighbors

    - -

    -Figure 5 -

    -
    -

    umap with recipes

    -
    -
    library(embed)
-set.seed(1234)
-
-recipe(mpg ~ ., data = mtcars) |>
-  step_umap(all_numeric_predictors()) |>
-  prep() |>
-  bake(NULL)
-#> # A tibble: 32 × 3
-#>      mpg  UMAP1 UMAP2
-#>    <dbl>  <dbl> <dbl>
-#>  1  21    1.50   2.26
-#>  2  21    1.26   2.02
-#>  3  22.8  2.17   4.19
-#>  4  21.4 -1.38  -1.67
-#>  5  18.7 -2.67  -2.66
-#>  6  18.1 -0.759 -1.57
-#>  7  14.3 -3.19  -3.37
-#>  8  24.4  1.31   3.91
-#>  9  22.8  2.06   2.46
-#> 10  19.2  0.876  2.31
-#> # ℹ 22 more rows
    -
    -
    -
    -

    Isomap

    -

    Isometric mapping is a non-linear dimensionality reduction method.

    -

    This is another method that uses distances between points to produce graphs of neighboring points. Where this method is different than other methods is that it uses geodesic distances as opposed to straight-line distances.

    -

    The geodesic distance is the sum of edge weights along the shortest path between two points.

    -

    The eigenvectors of the deodesic distance metric are then used to represent the new coordinates.

    -
    -
    -

    Isomap Algorithm

    -

    A very high-level description of the Isomap algorithm is given below.

    -
      -
    1. Find the neighbors for each point
      2. -
    2. Construct the neighborhood graph, using Euclidean distance as edge length
      3. -
    3. Calculate the shortest path between each pair of points
      4. -
    4. Use Multidimensional scaling to compute a lower-dimensional embedding
      5. -
    -
    -
    -

    Isomap Pros and Cons

    -
    -
    -

    Pros

    -
      -
    • Captures non-linear effects
      • -
    • Captures long-range structure, not just local structure
      • -
    • No parameters to set other than neighbors
      • -
    -
    -

    Cons

    -
      -
    • Computationally expensive
      • -
    • Assumes a single connected manifold
      • -
    -
    -
    -
    -

    isomap with recipes

    -
    -
    library(embed)
-set.seed(1234)
-
-recipe(mpg ~ ., data = mtcars) |>
-  step_isomap(all_numeric_predictors(), neighbors = 10) |>
-  prep() |>
-  bake(NULL)
-#> # A tibble: 32 × 6
-#>      mpg Isomap1 Isomap2 Isomap3 Isomap4 Isomap5
-#>    <dbl>   <dbl>   <dbl>   <dbl>   <dbl>   <dbl>
-#>  1  21    -98.5     35.6    35.3   137.    105. 
-#>  2  21    -98.5     35.6    35.1   137.    104. 
-#>  3  22.8 -161.      48.1    20.4   160.    129. 
-#>  4  21.4    8.55    14.6    37.2    86.8   125. 
-#>  5  18.7  136.      28.5    35.7   -30.8   102. 
-#>  6  18.1  -27.7     21.7    38.3   104.    116. 
-#>  7  14.3  207.     -16.5    34.6   -55.5   158. 
-#>  8  24.4 -147.      44.2    23.7   182.     88.4
-#>  9  22.8 -125.      41.1    18.1   151.    118. 
-#> 10  19.2  -91.6     33.7    48.6   135.    101. 
-#> # ℹ 22 more rows
    -
    - - -
    -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-area-vs-equivalent-diameter-1.svg b/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-area-vs-equivalent-diameter-1.svg deleted file mode 100644 index ff9fa35a..00000000 --- a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-area-vs-equivalent-diameter-1.svg +++ /dev/null @@ -1,1994 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -250 -500 -750 -1000 -1250 - - - - - - - - - - - -0 -250000 -500000 -750000 -1000000 -1250000 -area -equivalent_diameter - - - diff --git a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-mean-red-val-vs-mean-blue-val-1.svg b/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-mean-red-val-vs-mean-blue-val-1.svg deleted file mode 100644 index 2af6bf14..00000000 --- a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-mean-red-val-vs-mean-blue-val-1.svg +++ /dev/null @@ -1,1990 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 -50 -100 -150 -200 - - - - - - - - - - -0 -50 -100 -150 -200 -mean_red_val -mean_blue_val - - - diff --git a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-physiological-length-vs-diameter-1.svg b/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-physiological-length-vs-diameter-1.svg deleted file mode 100644 index b0b55d05..00000000 --- a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-physiological-length-vs-diameter-1.svg +++ /dev/null @@ -1,1970 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1000 -1500 - - - - -1000 -1500 -physiological_length -diameter - - - diff --git a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-umap-known-1.svg b/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-umap-known-1.svg deleted file mode 100644 index afb4b314..00000000 --- a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-umap-known-1.svg +++ /dev/null @@ -1,682 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -5 - - - - - - - - - - -10 - - - - - - - - - - -2 - - - - - - - - - - -3 - - - - - - - --2 --1 -0 -1 -2 - - - - - --2 --1 -0 -1 -2 - - - - --10 --5 -0 -5 - - - - - - --2 --1 -0 -1 -2 -3 --2 -0 -2 - - - --2 --1 -0 -1 -2 - - - - - --10 --5 -0 -5 -10 - - - - - --3 --2 --1 -0 -1 -2 -3 - - - - - - - -UMAP1 -UMAP2 - - - diff --git a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-umap-unknown-1.svg b/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-umap-unknown-1.svg deleted file mode 100644 index 0143a62f..00000000 --- a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/fig-umap-unknown-1.svg +++ /dev/null @@ -1,682 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -C - - - - - - - - - - -D - - - - - - - - - - -A - - - - - - - - - - -B - - - - - - - --2 --1 -0 -1 -2 - - - - - --2 --1 -0 -1 -2 - - - - --10 --5 -0 -5 - - - - - - --2 --1 -0 -1 -2 -3 --2 -0 -2 - - - --2 --1 -0 -1 -2 - - - - - --10 --5 -0 -5 -10 - - - - - --3 --2 --1 -0 -1 -2 -3 - - - - - - - -UMAP1 -UMAP2 - - - diff --git a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/pca-cumulative-percent-variance-1.svg b/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/pca-cumulative-percent-variance-1.svg deleted file mode 100644 index fb9b3054..00000000 --- a/docs/slides/advanced-05-feature-engineering-part-one_files/figure-revealjs/pca-cumulative-percent-variance-1.svg +++ /dev/null @@ -1,287 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -5 -0 -10 -20 -30 -40 -50 -5 -0 -10 -20 -30 -5 -10 -5 -0 -20 -40 -60 -0 -20 -40 -60 -component -Percent variance - - - diff --git a/docs/slides/advanced-05-feature-engineering-part-two.html b/docs/slides/advanced-05-feature-engineering-part-two.html index 4f515276..f038c306 100644 --- a/docs/slides/advanced-05-feature-engineering-part-two.html +++ b/docs/slides/advanced-05-feature-engineering-part-two.html @@ -7,8 +7,8 @@ - - + + Machine learning with tidymodels – 5 - Feature engineering: splines, target encoding and dates @@ -29,8 +29,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ + html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } - pre > code.sourceCode > span { line-height: 1.25; } + pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -41,7 +42,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } - pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } + pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -94,7 +95,7 @@ code span.vs { color: #20794d; } /* VerbatimString */ code span.wa { color: #5e5e5e; font-style: italic; } /* Warning */ - + @@ -1437,190 +1438,172 @@

    Datetime features

    + }); + \ No newline at end of file diff --git a/docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-1.svg b/docs/slides/advanced-05-feature-engineering-part-two_files/figure-revealjs/step-date-1.svg similarity index 100% rename from docs/slides/advanced-04-feature-engineering-part-two_files/figure-revealjs/fig-step-date-1.svg rename to docs/slides/advanced-05-feature-engineering-part-two_files/figure-revealjs/step-date-1.svg diff --git a/docs/slides/advanced-06-postprocessing.html b/docs/slides/advanced-06-postprocessing.html new file mode 100644 index 00000000..c87e9eaa --- /dev/null +++ b/docs/slides/advanced-06-postprocessing.html @@ -0,0 +1,1146 @@ + + + + + + + + + + + + + Machine learning with tidymodels – 6 - Postprocessing + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + +
    +

    6 - Postprocessing

    +

    Getting More Out of Feature Engineering and Tuning for Machine Learning

    + +
    +
    + +
    +
    +

    Startup!

    +
    +
    library(tidymodels)
+library(desirability2)
+library(probably)
+library(mirai)
+
+# check torch:
+if (torch::torch_is_installed()) {
+  library(torch)
+}
+
+tidymodels_prefer()
+theme_set(theme_bw())
+options(pillar.advice = FALSE, pillar.min_title_chars = Inf)
+daemons(parallel::detectCores())
    +
    +
    +
    +

    More startup!

    +
    +
    # Load our example data for this section
+"https://raw.githubusercontent.com/tidymodels/" |> 
+  paste0("workshops/main/slides/class_data.RData") |> 
+  url() |> 
+  load()
+
+set.seed(429)
+sim_split <- initial_split(class_data, prop = 0.75, strata = class)
+sim_train <- training(sim_split)
+sim_test  <- testing(sim_split)
+
+set.seed(523)
+sim_rs <- vfold_cv(sim_train, v = 10, strata = class)
    +
    +
    +
    +

    Our neural network model

    +
    +
    rec <- 
+  recipe(class ~ ., data = sim_train) |> 
+  step_normalize(all_numeric_predictors())
+
+nnet_spec <- 
+  mlp(hidden_units = tune(), penalty = tune(), learn_rate = tune(), 
+      epochs = 100, activation = tune()) |> 
+  # Remove the class_weights argument
+  set_engine("brulee", stop_iter = 10) |> 
+  set_mode("classification")
+  
+nnet_wflow <- workflow(rec, nnet_spec)
+
+nnet_param <- 
+  nnet_wflow |> 
+  extract_parameter_set_dials()
    +
    +
    +
    +
    +

    What is postprocessing?

    + +
    +
    +

    Adjusting model predictions

    +

    How can we modify our predictions? Some examples:

    +
      +
    • Fixing calibration issues*.
      • +
    • Limit the range of predictions.
      • +
    • Alternative cutoffs for binary data.
      • +
    • Declining to predict.
      • +
    +
    +

    * Requires further estimation (and data).

    +


    +

    Let’s first consider the easiest case: alternative cutoffs.

    +
    +
    +
    +

    Alternative thresholds

    +

    Instead of up-weighting the samples in the minority (via class_weights), we can try to fit the best model and then define what it means to be an “event.”

    +


    +

    Instead of using a 50% threshold, we might lower the level of evidence needed to call a prediction an event.

    +


    +

    How do we tune the threshold?

    +
    +
    +

    Tailors

    +

    The tailor package is similar to recipes but specifies how to adjust predictions.

    +

    A simple example:

    +
    +
    thrsh_tlr <-
+  tailor() |>
+  adjust_probability_threshold(threshold = 1 / 3)
+
+thrsh_tlr
    +
    +
      +

    • Like a recipe, this initial call doesn’t do anything but declare intent.

      • +

    • Unlike a recipe, it does not need the data (i.e., predictions) at this point.

      +
        +
      • Relevant prediction columns are selected when fit() is used (next slide).
        • +
      • +
    +
    +
    +

    Manual use of a tailor

    +

    There is a fit() method that requires data and the names of the prediction columns:

    +
    +
    three_rows <- 
+  tribble(
+     ~ class, ~ .pred_class, ~.pred_event, ~.pred_nonevent,
+     "event",       "event",          0.6,             0.4,
+     "event",    "nonevent",          0.4,             0.6, 
+  "nonevent",    "nonevent",          0.1,             0.9  
+  ) |> 
+  mutate(across(where(is.character), factor))
+
+thrsh_fit <-
+  thrsh_tlr |>
+  fit(
+    three_rows,
+    outcome = class,
+    estimate = .pred_class,
+    .pred_event:.pred_nonevent  # No argument name and order matches factor levels
+  )
    +
    +
    +
    +

    Manual use of a tailor

    +

    predict() applies the adjustments:

    +
    +
    thrsh_fit
+
+predict(thrsh_fit, three_rows)
+#> # A tibble: 3 × 4
+#>   class    .pred_class .pred_event .pred_nonevent
+#>   <fct>    <fct>             <dbl>          <dbl>
+#> 1 event    event               0.6            0.4
+#> 2 event    event               0.4            0.6
+#> 3 nonevent nonevent            0.1            0.9
    +
    +


    +
    +
    +

    tailors within workflows

    +

    In practice, we would add the tailor to a workflow to make it easier to use:

    +


    +
    +
    nnet_wflow <- workflow(rec, nnet_spec, thrsh_tlr)
    +
    +


    +
      +
    • We don’t have to set the names of the outcome or prediction columns (yet).
      • +
    • fit() and predict() happen automatically.
      • +
    +
    +
    +

    Current adjustments

    + +
    +
    +

    Some notes

    +
      +

    • Adjustment order matters; tailor will error early if the ordering rules are violated.

      • +

    • Adjustments that change class probabilities also affect hard class predictions.

      • +

    • Adjustments happen before performance estimation.

      +
        +
      • Undoing something like a log transformation is a bad idea here.
        • +
      • +

    • We have more calibration methods in mind.

      • +
    +
    +
    +

    Your turn

    +
      +
    • Discuss with those around you what the “ordering rules” could be.
      • +
    +


    +
    +
    +
    +
    +03:00 +
    +
    +
    +
    +
    +

    More Notes

    +
      +

    • When estimation is required, the data considerations become more complex.

      • +

    • Most arguments can be tuned.

      • +

    • For grid search, we use a conditional execution algorithm that avoids redundant retraining of the preprocessor or model.

      • +
    +
    +
    +
    +

    Back to our neural network

    + +
    +
    +

    Tuning the probability threshold

    +
    +
    thrsh_tlr <-
+  tailor() |>
+  adjust_probability_threshold(threshold = tune())
+
+nnet_thrsh_wflow <- workflow(rec, nnet_spec, thrsh_tlr)
+  
+nnet_thrsh_param <- 
+  nnet_thrsh_wflow |> 
+  extract_parameter_set_dials() |> 
+  update(threshold = threshold(c(0.001, 0.5)))
    +
    +
    +
    +

    Tuning the probability threshold

    +

    Nearly the same code as before:

    +


    +
    +
    ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)
+cls_mtr <- metric_set(brier_class, roc_auc, sensitivity, specificity)
+
+set.seed(12)
+nnet_thrsh_res <-
+  nnet_thrsh_wflow |>
+  tune_grid(
+    resamples = sim_rs,
+    grid = 25,
+    param_info = nnet_thrsh_param, 
+    control = ctrl,
+    metrics = cls_mtr
+  )
    +
    +
    +
    +

    Grid results

    +
    +
    autoplot(nnet_thrsh_res)
    +
    +
    +
    +

    +
    +
    +
    +
    +
    +
    +

    Grid results

    +
      +
    • tanh activation is doing much better.
      • +
    • threshold should not (and does not) affect the Brier or ROC metrics.
      • +
    • We can achieve low Brier scores.
      • +
    • We could run another grid with values <2% for a better threshold estimate.
      • +
    +
    +
    +

    Multimetric optimization

    +
    +
    nnet_thrsh_res |>
+  show_best_desirability(
+    maximize(sensitivity),
+    minimize(brier_class),
+    constrain(specificity, low = 0.8, high = 1.0)
+  ) |>
+  relocate(threshold, sensitivity, specificity, brier_class, .d_overall)
+#> # A tibble: 5 × 14
+#>   threshold sensitivity specificity brier_class .d_overall hidden_units  penalty
+#>       <dbl>       <dbl>       <dbl>       <dbl>      <dbl>        <int>    <dbl>
+#> 1    0.0218       0.935       0.870      0.0449      0.948           36 2.61e- 5
+#> 2    0.250        0.828       0.951      0.0418      0.902           28 2.61e-10
+#> 3    0.209        0.833       0.936      0.0436      0.896            8 1   e-10
+#> 4    0.188        0.833       0.934      0.0445      0.891           18 5.62e- 2
+#> 5    0.0634       0.899       0.890      0.0527      0.884           40 2.15e- 2
+#> # ℹ 7 more variables: activation <chr>, learn_rate <dbl>, .config <chr>,
+#> #   roc_auc <dbl>, .d_max_sensitivity <dbl>, .d_min_brier_class <dbl>,
+#> #   .d_box_specificity <dbl>
    +
    +
    +
    +

    Calibration

    +
    +
    +
    more_sens <-
+  nnet_thrsh_res |>
+  select_best_desirability(
+    maximize(sensitivity),
+    minimize(brier_class),
+    constrain(specificity, low = 0.8, high = 1.0)
+  )
+
+nnet_thrsh_res |>
+  collect_predictions(
+    parameters = more_sens
+  ) |>
+  cal_plot_windowed(
+    truth = class,
+    estimate = .pred_event,
+    window_size = 0.2,
+    step_size = 0.025,
+  )
    +
    +
    +
    +
    +

    +
    +
    +
    +
    +
    +
    +

    Thoughts about these results

    +

    The calibration issue in the previous plot shows that some very likely non-events will have underestimated probabilities.

    +
      +
    • That may not matter if we are very focused on events.
      • +
    • Thresholding does not affect calibration.
      • +
    • We might be able to: +
        +
      • Further tune the neural network to solve the issue and/or
        • +
      • Add a calibration postprocessor
        • +
      • +
    +
    +
    +

    Thoughts about the approach

    +

    Let’s say that we pick a threshold of 2%. Our explanation to the user/stakeholder would be

    +
    +

    “As long as the model is at least 2% sure it is an event, we will call it an event”.

    +
    +

    It may be challenging to convince someone that this is the best option.

    +


    +

    That said, this is probably a better approach than cost-sensitive learning.

    +
    +
    +
    +

    Fitting a postprocessor

    + +
    +
    +

    Data to train the adjustments

    +

    If an adjustment requires data, where do we get it from?

    +
      +

    • Fitting a calibration model to the training set re-predictions would be bad.

      • +

    • Also, we don’t want to touch the validation or test sets.

      • +
    +
    +


    +

    We need another data set.

    +
    +
    +
    +

    Data sources

    +

    Two possibilities:

    +
      +
    1. Shave some data off the training set to create a calibration set. +
        +
      • During resampling, we can do the same to the analysis set.
        • +
      • The “shaving” process emulates the original sampling method.
        • +
      • There are fewer data points for training the preprocessor and primary model.
        • +
      2. +
    2. Use a static calibration set outside our training/validation/testing splits.
      3. +
    +

    Currently, we have implemented the first method.

    +
    +
    +

    Example: 3-fold CV

    + +
    +
    +

    Example: 3-fold CV internal split

    + +
    +
    +

    Breakdown for the class imbalance data

    +


    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    DataStrategyeventno_event
    OriginalAll2251775
    TrainingNo Calibration1681331
    AnalysisNo Calibration1511197
    TrainingCalibration126998
    AnalysisCalibration1351077
    CalibrationCalibration16120
    +
    +
    +
    +
    +

    Calibration

    +

    Calibration models, in essence, try to predict the true class using the model predictions. Symbolically:

    +
      +
    • For regression models: outcome ~ .pred
      • +
    • For binary classifiers: class ~ .pred_class_1
      • +
    • For multiclass: class ~ .pred_class_1 + .pred_class_2 + ...
      • +
    +

    Each calibration method works slightly differently.

    +

    For example, in regression, a (generalized) linear model is fit, and the residuals are added to new predictions.

    +
    +
    +

    Calibration expectations

    +

    Keep expectations low. For these methods to work:

    +
      +
    • The systematic issue will need to be large, or at least not subtle.
      • +
    • A large calibration set is needed to work effectively.
      • +
    +

    The example in ALM4TD is an illustrative example and details.

    +

    In many cases, trying a different model or tuning parameters would be better.

    +


    +

    You can tune the calibration method, one of which is no calibration.

    +
    +
    +

    Your turn

    +
      +
    • Based on previous results, choose and fix a specific activation type (i.e., no tune()).
      • +
    • Add a calibrator to your tailor with a method = tune() value.
      • +
    • Run another grid search
      • +
    +

    Does it help with this data set?

    +


    +
    +
    +
    +
    +10:00 +
    +
    +
    + + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/slides/advanced-06-postprocessing_files/figure-revealjs/nnet-cal-plot-sens-1.svg b/docs/slides/advanced-06-postprocessing_files/figure-revealjs/nnet-cal-plot-sens-1.svg new file mode 100644 index 00000000..7a39e320 --- /dev/null +++ b/docs/slides/advanced-06-postprocessing_files/figure-revealjs/nnet-cal-plot-sens-1.svg @@ -0,0 +1,1637 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +0.00 +0.25 +0.50 +0.75 +1.00 + + + + + + + + + + +0.00 +0.25 +0.50 +0.75 +1.00 +Window Midpoint +Event Rate + + + diff --git a/docs/slides/advanced-06-postprocessing_files/figure-revealjs/sim-nnet-threshold-1.svg b/docs/slides/advanced-06-postprocessing_files/figure-revealjs/sim-nnet-threshold-1.svg new file mode 100644 index 00000000..3ed67247 --- /dev/null +++ b/docs/slides/advanced-06-postprocessing_files/figure-revealjs/sim-nnet-threshold-1.svg @@ -0,0 +1,1037 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# Hidden Units + + + + + + + + + + +Amount of Regularization (log-10) + + + + + + + + + + +Learning Rate (log-10) + + + + + + + + + + +Threshold + + + + + + + + + + +brier_class + + + + + + + + + + +roc_auc + + + + + + + + + + +sensitivity + + + + + + + + + + +specificity + + + + + + + + +0 +10 +20 +30 +40 +50 + + + + + +-10.0 +-7.5 +-5.0 +-2.5 +0.0 + + + +-3 +-2 +-1 + + + + + + +0.0 +0.1 +0.2 +0.3 +0.4 +0.5 +0.04 +0.06 +0.08 +0.10 + + + + +0.7 +0.8 +0.9 + + + +0.4 +0.6 +0.8 +1.0 + + + + +0.4 +0.6 +0.8 + + + + +Activation Function + + + + + + + + + + +elu +log_sigmoid +relu +tanh +tanhshrink + + + diff --git a/docs/slides/advanced-07-feature-selection.html b/docs/slides/advanced-07-feature-selection.html new file mode 100644 index 00000000..7ce35a23 --- /dev/null +++ b/docs/slides/advanced-07-feature-selection.html @@ -0,0 +1,1096 @@ + + + + + + + + + + + + + Machine learning with tidymodels – 7 - Feature selection + + + + + + + + + + + + + + + + + + + + + +
    +
    + +
    +

    7 - Feature selection

    +

    Getting More Out of Feature Engineering and Tuning for Machine Learning

    + +
    +
    + +
    +
    +

    Startup!

    +
    +
    library(tidymodels)
+library(important)
+library(probably)
+library(mirai)
+
+tidymodels_prefer()
+theme_set(theme_bw())
+options(pillar.advice = FALSE, pillar.min_title_chars = Inf)
+daemons(parallel::detectCores())
    +
    +
    +
    +

    More startup!

    +
    +
    # Load our example data for this section
+"https://raw.githubusercontent.com/tidymodels/" |> 
+  paste0("workshops/main/slides/class_data.RData") |> 
+  url() |> 
+  load()
+
+set.seed(429)
+sim_split <- initial_split(class_data, prop = 0.75, strata = class)
+sim_train <- training(sim_split)
+sim_test  <- testing(sim_split)
+
+set.seed(523)
+sim_rs <- vfold_cv(sim_train, v = 10, strata = class)
    +
    +
    +
    +
    +

    Why remove features/predictors?

    + +
    +
    +

    Models and feature selection

    +

    Some models automatically remove predictors by never using them in the model:

    +
      +
    • tree- and rule-based models
      • +
    • some regularized models (e.g., glmnet)
      • +
    • multivariate adaptive regression splines (MARS)
      • +
    • RuleFit
      • +
    • not really ensembles though
      • +
    +

    Sometimes using irrelevant predictors hurts model performance.

    +
    +
    +

    Effects of extra predictors

    + +
    +

    AML4TD

    +
    +
    +
    +

    General selection methods

    +
      +

    • wrappers: a sequential algorithm proposes feature subsets, fits the model with these subsets, and then determines a better subset from the results.

      • +

    • filters: screen predictors before adding them to the model.

      • +
    +

    tidymodels doesn’t have any wrappers (but see the caret documentation for them)

    +


    +

    The new important package does have filters via recipes.

    +
    +
    +

    Be careful!!!

    +

    tidymodels has always contained some “hidden guardrails” that should prevent practitioners from making subtle (but consequential) methodological mistakes.

    +


    +

    Feature selection is a good example. Based on the literature, it is easily done wrong.

    +


    +

    The selection process should take place inside a resampling loop so that the workflow does not overfit the predictors.

    +
    +

    A&M (2002), AML4TD, FES

    +
    +
    +
    +
    +

    Imbalanced example (again)

    + +
    +
    +

    IMPORTANT

    +

    We released two packages this year that enable supervised feature selection:

    +
      +
    • filtro: low-level scoring methods for predictors (e.g., importance).
      • +
    • important: tools for permutation importance and recipes steps for supervised feature selection.
      • +
    +


    +

    Let’s look at the help page for important::step_predictor_best().

    +
    +
    +

    K-nearest neighbors

    +
    +
    rec <-
+  recipe(class ~ ., data = sim_train) |>
+  step_predictor_best(
+    all_predictors(),
+    score = "imp_rf",
+    prop_terms = tune(),
+    id = "filter"
+  ) |>
+  step_normalize(all_numeric_predictors())
+  
+knn_spec <- 
+  nearest_neighbor(neighbors = tune(), weight_func = tune()) |> 
+  set_mode("classification")
+  
+thrsh_tlr <-
+  tailor() |>
+  adjust_probability_threshold(threshold = tune())
    +
    +
    +
    +

    Setup the workflow

    +
    +
    knn_wflow <- workflow(rec, knn_spec, thrsh_tlr)
+
+knn_param <-
+  knn_wflow |>
+  extract_parameter_set_dials() |>
+  update(
+    threshold = threshold(c(0.001, 0.1)),
+    neighbors = neighbors(c(1, 50))
+  )
    +
    +
    +
    +

    Tuning results

    +
    +
    cls_mtr <- metric_set(brier_class, roc_auc, sensitivity, specificity)
+ctrl <- control_grid(save_pred = TRUE, save_workflow = TRUE)
+
+set.seed(12)
+knn_res <-
+  knn_wflow |>
+  tune_grid(
+    resamples = sim_rs,
+    grid = 50,
+    control = ctrl,
+    metrics = cls_mtr,
+    param_info = knn_param
+  )
    +
    +
    +
    +

    Grid results

    +
    +
    autoplot(knn_res)
    +
    +
    +
    +

    +
    +
    +
    +
    +
    +
    +

    Brier results

    +
    +
    autoplot(knn_res, metric = "brier_class") + 
+  facet_grid(. ~ name, scale = "free_x")
    +
    +
    +
    +

    +
    +
    +
    +
    +
    +
    +

    ROC curve results

    +
    +
    autoplot(knn_res, metric = "roc_auc") + 
+  facet_grid(. ~ name, scale = "free_x")
    +
    +
    +
    +

    +
    +
    +
    +
    +
    +
    +

    Sensitivity/Specificity results

    +
    +
    autoplot(knn_res, metric = c("sensitivity", "specificity"))
    +
    +
    +
    +

    +
    +
    +
    +
    +
    +
    +

    Fit the model and get filter information

    +
    +
    knn_fit <- fit_best(knn_res, metric = "brier_class")
+filter_info <-
+    knn_fit |>
+    extract_recipe() |>
+    tidy(id = "filter")
+
+filter_info
+#> # A tibble: 30 × 4
+#>    terms        removed      score id    
+#>    <chr>        <lgl>        <dbl> <chr> 
+#>  1 predictor_01 TRUE     0.00152   filter
+#>  2 predictor_02 TRUE     0.00170   filter
+#>  3 predictor_03 TRUE    -0.0000460 filter
+#>  4 predictor_04 TRUE     0.000968  filter
+#>  5 predictor_05 TRUE    -0.0000626 filter
+#>  6 predictor_06 TRUE     0.000126  filter
+#>  7 predictor_07 TRUE     0.000779  filter
+#>  8 predictor_08 TRUE     0.00160   filter
+#>  9 predictor_09 TRUE     0.00218   filter
+#> 10 predictor_10 TRUE    -0.000302  filter
+#> # ℹ 20 more rows
    +
    +
    +
    +

    The truth about our data

    +

    The data were simulated and 15 out of 30 predictors were uninformative (and highly correlated). How did we do?

    +
    +


    +
    +
    +
    +
    + + + + + + + + + + + + + + + + + + + + +
    noisereal
    kept02
    removed1513
    +
    +
    +
    +
      +
    • selection sensitivity: 13.3%
      • +
    • selection specificity: 100%
      • +
    +

    It was good at removing noise but not keeping the real predictors.

    +
    +
    +
    +
    +

    Random forest importance scores

    +
    +
    +
    # A "truth" column was added
+filter_info |>
+  mutate(
+    terms = factor(terms),
+    terms = reorder(terms, score)
+  ) |>
+  ggplot(
+    aes(x = score, 
+        y = terms, 
+        fill = truth)
+  ) +
+  geom_bar(stat = "identity") + 
+  labs(x = "RF Importance", y = NULL) + 
+  scale_fill_brewer(palette = "Set2")
    +
    +
    +
    +
    +

    +
    +
    +
    +
    +
    +
    +

    The simulation

    +

    The simulation system is documented here with method = "caret". The two most important predictors being retained correspond to:

    +
    +
    # In logit units: 
+- 4 * two_factor_1 + 4 * two_factor_2 + 2 * two_factor_1 * two_factor_2
    +
    +


    +

    Most of the others are small linear effects and tree-based models are not great at modeling those.

    +


    +

    Also, the noise predictors were simulated to have fairly high correlations with one another. That can often compromise random forest importance scores.

    +
    +
    +

    Other steps

    +

    The important package has two other feature selection steps that can be used with multiple scores:

    +
    + +
    imp_rf > 2 & cor_pearson >= 0.75
    +
    +
    + +
    desirability(
+  maximize(correlation),
+  maximize(imp_rf)
+)
    +
    +
    +
    +
    +

    Proceed to the test set

    + +
    +
    +

    Manual approach

    +

    We already have our fitted model and, if we are happy with it:

    +
    +
    test_pred <- augment(knn_fit, sim_test)
+test_pred |> cls_mtr(class, estimate = .pred_class, .pred_event)
+#> # A tibble: 4 × 3
+#>   .metric     .estimator .estimate
+#>   <chr>       <chr>          <dbl>
+#> 1 sensitivity binary        0.982 
+#> 2 specificity binary        0.874 
+#> 3 brier_class binary        0.0246
+#> 4 roc_auc     binary        0.981
    +
    +
    +

    Resampling estimates:

    +
    +
    #> # A tibble: 4 × 4
+#>   .metric       mean     n std_err
+#>   <chr>        <dbl> <int>   <dbl>
+#> 1 sensitivity 0.958     10 0.0154 
+#> 2 specificity 0.867     10 0.00948
+#> 3 brier_class 0.0349    10 0.00201
+#> 4 roc_auc     0.968     10 0.00714
    +
    +
    +
    +
    +

    Checking (Approximate) Calibration

    +
    +
    +
    +
    test_pred|>
+  cal_plot_windowed(
+    truth = class,
+    estimate = .pred_event,
+    window_size = 0.2,
+    step_size = 0.025,
+  )
    +
    +


    +

    Looks alright. The small effective sample size (57 events) makes it pretty noisy.

    +
    +
    +
    +
    +
    +

    +
    +
    +
    +
    +
    +
    +
    +

    Automated approach

    +

    Similar to fit_best(), there is a convenience function that can be used to get the final model and the test set results.

    +


    +

    We have to start with a finalized workflow (i.e., no tune() values):

    +
    +
    knn_best <- select_best(knn_res, metric = "brier_class")
+knn_last_wflow <- finalize_workflow(knn_wflow, knn_best)
    +
    +
    +
    +

    Automated approach

    +

    last_fit() uses the original split object to fit, predict, and measure the model using the test set:

    +
    +
    knn_test_res <- 
+  knn_last_wflow |> 
+  last_fit(sim_split, metrics = cls_mtr)
+  
+knn_test_res
+#> # Resampling results
+#> # Manual resampling 
+#> # A tibble: 1 × 6
+#>   splits             id               .metrics .notes   .predictions .workflow 
+#>   <list>             <chr>            <list>   <list>   <list>       <list>    
+#> 1 <split [1499/501]> train/test split <tibble> <tibble> <tibble>     <workflow>
    +
    +
    +
    +

    Automated approach

    +

    We can pick out the parts that we want:

    +
    +
    knn_final_fit <- knn_test_res |> extract_workflow()
+knn_test_pred <- knn_test_res |> collect_predictions()
+knn_test_mtr  <- knn_test_res |> collect_metrics()
+
+knn_test_mtr
+#> # A tibble: 4 × 4
+#>   .metric     .estimator .estimate .config        
+#>   <chr>       <chr>          <dbl> <chr>          
+#> 1 sensitivity binary        0.982  pre0_mod0_post0
+#> 2 specificity binary        0.874  pre0_mod0_post0
+#> 3 brier_class binary        0.0246 pre0_mod0_post0
+#> 4 roc_auc     binary        0.981  pre0_mod0_post0
    +
    +

    Easy peasy!

    + + +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/slides/advanced-07-feature-selection_files/figure-revealjs/knn-cal-plot-manual-1.svg b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/knn-cal-plot-manual-1.svg new file mode 100644 index 00000000..c9402c8f --- /dev/null +++ b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/knn-cal-plot-manual-1.svg @@ -0,0 +1,639 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +0.00 +0.25 +0.50 +0.75 +1.00 + + + + + + + + + + +0.00 +0.25 +0.50 +0.75 +1.00 +Window Midpoint +Event Rate + + + diff --git a/docs/slides/advanced-07-feature-selection_files/figure-revealjs/rf-imp-1.svg b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/rf-imp-1.svg new file mode 100644 index 00000000..7cb700fe --- /dev/null +++ b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/rf-imp-1.svg @@ -0,0 +1,184 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +predictor_10 +predictor_05 +predictor_03 +predictor_17 +predictor_23 +predictor_19 +predictor_25 +predictor_26 +predictor_06 +predictor_20 +predictor_22 +predictor_14 +predictor_21 +predictor_18 +predictor_07 +predictor_04 +predictor_11 +predictor_16 +predictor_15 +predictor_01 +predictor_08 +predictor_02 +predictor_28 +predictor_13 +predictor_30 +predictor_12 +predictor_09 +predictor_24 +predictor_29 +predictor_27 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +0.00 +0.02 +0.04 +RF Importance + +truth + + + + +noise +real + + + diff --git a/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-all-1.svg b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-all-1.svg new file mode 100644 index 00000000..454c9276 --- /dev/null +++ b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-all-1.svg @@ -0,0 +1,1127 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# Nearest Neighbors + + + + + + + + + + +Proportion of Top Predictors + + + + + + + + + + +Threshold + + + + + + + + + + +brier_class + + + + + + + + + + +roc_auc + + + + + + + + + + +sensitivity + + + + + + + + + + +specificity + + + + + + + + +0 +10 +20 +30 +40 +50 + + + + +0.25 +0.50 +0.75 +1.00 + + + + + +0.000 +0.025 +0.050 +0.075 +0.100 +0.05 +0.07 +0.09 + + + +0.7 +0.8 +0.9 + + + +0.4 +0.6 +0.8 +1.0 + + + + +0.25 +0.50 +0.75 +1.00 + + + + + +Distance Weighting Function + + + + + + + + + + + + + + + + + + +biweight +cos +epanechnikov +gaussian +inv +rank +rectangular +triangular +triweight + + + diff --git a/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-brier-1.svg b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-brier-1.svg new file mode 100644 index 00000000..0528d63d --- /dev/null +++ b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-brier-1.svg @@ -0,0 +1,363 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# Nearest Neighbors + + + + + + + + + + +Proportion of Top Predictors + + + + + + + + + + +Threshold + + + + + + + + +0 +10 +20 +30 +40 +50 + + + + +0.25 +0.50 +0.75 +1.00 + + + + + +0.000 +0.025 +0.050 +0.075 +0.100 +0.05 +0.07 +0.09 + + + +brier_class + +Distance Weighting Function + + + + + + + + + + + + + + + + + + +biweight +cos +epanechnikov +gaussian +inv +rank +rectangular +triangular +triweight + + + diff --git a/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-roc-1.svg b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-roc-1.svg new file mode 100644 index 00000000..a858f2ef --- /dev/null +++ b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-roc-1.svg @@ -0,0 +1,363 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# Nearest Neighbors + + + + + + + + + + +Proportion of Top Predictors + + + + + + + + + + +Threshold + + + + + + + + +0 +10 +20 +30 +40 +50 + + + + +0.25 +0.50 +0.75 +1.00 + + + + + +0.000 +0.025 +0.050 +0.075 +0.100 +0.7 +0.8 +0.9 + + + +roc_auc + +Distance Weighting Function + + + + + + + + + + + + + + + + + + +biweight +cos +epanechnikov +gaussian +inv +rank +rectangular +triangular +triweight + + + diff --git a/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-two-class-1.svg b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-two-class-1.svg new file mode 100644 index 00000000..ae56a744 --- /dev/null +++ b/docs/slides/advanced-07-feature-selection_files/figure-revealjs/sim-knn-two-class-1.svg @@ -0,0 +1,629 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# Nearest Neighbors + + + + + + + + + + +Proportion of Top Predictors + + + + + + + + + + +Threshold + + + + + + + + + + +sensitivity + + + + + + + + + + +specificity + + + + + + + + +0 +10 +20 +30 +40 +50 + + + + +0.25 +0.50 +0.75 +1.00 + + + + + +0.000 +0.025 +0.050 +0.075 +0.100 +0.4 +0.6 +0.8 +1.0 + + + + +0.25 +0.50 +0.75 +1.00 + + + + + +Distance Weighting Function + + + + + + + + + + + + + + + + + + +biweight +cos +epanechnikov +gaussian +inv +rank +rectangular +triangular +triweight + + + diff --git a/docs/slides/advanced-06-wrapping-up.html b/docs/slides/advanced-08-wrapping-up.html similarity index 96% rename from docs/slides/advanced-06-wrapping-up.html rename to docs/slides/advanced-08-wrapping-up.html index 58b2f1b4..e539276e 100644 --- a/docs/slides/advanced-06-wrapping-up.html +++ b/docs/slides/advanced-08-wrapping-up.html @@ -7,10 +7,10 @@ - - + + - Machine learning with tidymodels – 6 - Wrapping up + Machine learning with tidymodels – 9 - Wrapping up @@ -29,7 +29,7 @@ vertical-align: middle; } - + @@ -123,11 +123,11 @@ - - + + - - + + @@ -136,8 +136,8 @@
    -

    6 - Wrapping up

    -

    Advanced tidymodels

    +

    9 - Wrapping up

    +

    Getting More Out of Feature Engineering and Tuning for Machine Learning

    @@ -191,6 +191,11 @@

    Resources to keep learning

    + +
    +

    Follow us on Bluesky, Mastodon and at the tidyverse blog for updates!

    diff --git a/docs/slides/animations/anime_gp.gif b/docs/slides/animations/anime_gp.gif deleted file mode 100644 index 29662d28..00000000 Binary files a/docs/slides/animations/anime_gp.gif and /dev/null differ diff --git a/docs/slides/animations/anime_improvement.gif b/docs/slides/animations/anime_improvement.gif deleted file mode 100644 index 2a437e0a..00000000 Binary files a/docs/slides/animations/anime_improvement.gif and /dev/null differ diff --git a/docs/slides/annotations.html b/docs/slides/annotations.html index 79603745..5fa4e562 100644 --- a/docs/slides/annotations.html +++ b/docs/slides/annotations.html @@ -2,7 +2,7 @@ - + @@ -21,8 +21,9 @@ vertical-align: middle; } /* CSS for syntax highlighting */ +html { -webkit-text-size-adjust: 100%; } pre > code.sourceCode { white-space: pre; position: relative; } -pre > code.sourceCode > span { line-height: 1.25; } +pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } @@ -33,7 +34,7 @@ } @media print { pre > code.sourceCode { white-space: pre-wrap; } -pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; } +pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } @@ -64,15 +65,16 @@ - + + - + - + + }); +