# CoCalc, by SageMath, Inc., (c) 2016, 2017 -- License: AGPLv312###3# Webpack configuration file45Run dev server with source maps:67npm run webpack-watch89Then visit (say)1011https://dev0.sagemath.com/1213or for smc-in-smc project, info.py URL, e.g.1415https://cloud.sagemath.com/14eed217-2d3c-4975-a381-b69edcb40e0e/port/56754/1617This is far from ready to use yet, e.g., we need to properly serve primus websockets, etc.:1819webpack-dev-server --port=9000 -d2021Resources for learning webpack:2223- https://github.com/petehunt/webpack-howto24- http://webpack.github.io/docs/tutorials/getting-started/2526---2728## Information for developers2930This webpack config file might look scary, but it only consists of a few moving parts.31321. There is the "main" SMC application, which is split into "css", "lib" and "smc":331. css: a collection of all static styles from various locations. It might be possible34to use the text extraction plugin to make this a .css file, but that didn't work out.35Some css is inserted, but it doesn't work and no styles are applied. In the end,36it doesn't matter to load it one way or the other. Furthermore, as .js is even better,37because the initial page load is instant and doesn't require to get the compiled css styles.382. lib: this is a compilation of the essential js files in webapp-lib (via webapp-lib.coffee)393. smc: the core smc library. besides this, there are also chunks ([number]-hash.js) that are40loaded later on demand (read up on `require.ensure`).41For example, such a chunkfile contains latex completions, the data for the wizard, etc.422. There are static html files for the policies.43The policy files originate in webapp-lib/policies, where at least one file is generated by update_react_static.44That script runs part of the smc application in node.js to render to html.45Then, that html output is included into the html page and compiled.46It's not possible to automate this fully, because during the processing of these templates,47the "css" chunk from point 1.1 above is injected, too.48In the future, also other elements from the website (e.g. <Footer/>) will be rendered as49separate static html template elements and included there.503. There are auxiliary files for the "video chat" functionality. That might be redone differently, but51for now rendering to html only works via the html webpack plugin in such a way,52that it rewrites paths and post processes the files correctly to work.5354The remaining configuration deals with setting up variables (misc_node contains the centralized55information about where the page is getting rendered to, because also the hub.coffee needs to know56about certain file locations)5758Development vs. Production: There are two variables DEVMODE and PRODMODE.59* Prodmode:60* additional compression is enabled (do *not* add the -p switch to webpack, that's done here explicitly!)61* all output filenames, except for the essential .html files, do have hashes and a rather flat hierarchy.62* Devmode:63* Apply as little additional plugins as possible (compiles faster).64* File names have no hashes, or hashes are deterministically based on the content.65This means, when running webpack-watch, you do not end up with a growing pile of66thousands of files in the output directory.6768MathJax: It lives in its own isolated world. This means, don't mess with the MathJax.js ...69It needs to know from where it is loaded (the path in the URL), to retrieve many additional files on demand.70That's also the main reason why it is slow, because for each file a new SSL connection has to be setup!71(unless, http/2 or spdy do https pipelining).72How do we help MathJax a little bit by caching it, when the file names aren't hashed?73The trick is to add the MathJax version number to the path, such that it is unique and will definitely74trigger a reload after an update of MathJax.75The MathjaxVersionedSymlink below (in combination with misc_node.MATHJAX_LIB)76does extract the MathJax version number, computes the path, and symlinks to its location.77Why in misc_node? The problem is, that also the jupyter server (in its isolated iframe),78needs to know about the MathJax URL.79That way, the hub can send down the URL to the jupyter server (there is no webapp client in between).80###8182'use strict'8384_ = require('lodash')85webpack = require('webpack')86path = require('path')87fs = require('fs')88glob = require('glob')89child_process = require('child_process')90misc = require('smc-util/misc')91misc_node = require('smc-util-node/misc_node')92async = require('async')93program = require('commander')9495SMC_VERSION = require('smc-util/smc-version').version96theme = require('smc-util/theme')9798git_head = child_process.execSync("git rev-parse HEAD")99GIT_REV = git_head.toString().trim()100TITLE = theme.SITE_NAME101DESCRIPTION = theme.APP_TAGLINE102SMC_REPO = 'https://github.com/sagemathinc/cocalc'103SMC_LICENSE = 'AGPLv3'104WEBAPP_LIB = misc_node.WEBAPP_LIB105INPUT = path.resolve(__dirname, WEBAPP_LIB)106OUTPUT = misc_node.OUTPUT_DIR107DEVEL = "development"108NODE_ENV = process.env.NODE_ENV || DEVEL109PRODMODE = NODE_ENV != DEVEL110CDN_BASE_URL = process.env.CDN_BASE_URL # CDN_BASE_URL must have a trailing slash111DEVMODE = not PRODMODE112MINIFY = !! process.env.WP_MINIFY113DEBUG = '--debug' in process.argv114SOURCE_MAP = !! process.env.SOURCE_MAP115STATICPAGES = !! process.env.CC_STATICPAGES # special mode where just the landing page is built116date = new Date()117BUILD_DATE = date.toISOString()118BUILD_TS = date.getTime()119GOOGLE_ANALYTICS = misc_node.GOOGLE_ANALYTICS120121# create a file base_url to set a base url122BASE_URL = misc_node.BASE_URL123124# check and sanitiziation (e.g. an exising but empty env variable is ignored)125# CDN_BASE_URL must have a trailing slash126if not CDN_BASE_URL? or CDN_BASE_URL.length == 0127CDN_BASE_URL = null128else129if CDN_BASE_URL[-1..] isnt '/'130throw new Error("CDN_BASE_URL must be an URL-string ending in a '/' -- but it is #{CDN_BASE_URL}")131132# output build environment variables of webpack133console.log "SMC_VERSION = #{SMC_VERSION}"134console.log "SMC_GIT_REV = #{GIT_REV}"135console.log "NODE_ENV = #{NODE_ENV}"136console.log "BASE_URL = #{BASE_URL}"137console.log "CDN_BASE_URL = #{CDN_BASE_URL}"138console.log "DEBUG = #{DEBUG}"139console.log "MINIFY = #{MINIFY}"140console.log "INPUT = #{INPUT}"141console.log "OUTPUT = #{OUTPUT}"142console.log "GOOGLE_ANALYTICS = #{GOOGLE_ANALYTICS}"143144# mathjax version → symlink with version info from package.json/version145if CDN_BASE_URL?146# the CDN url does not have the /static/... prefix!147MATHJAX_URL = CDN_BASE_URL + path.join(misc_node.MATHJAX_SUBDIR, 'MathJax.js')148else149MATHJAX_URL = misc_node.MATHJAX_URL # from where the files are served150MATHJAX_ROOT = misc_node.MATHJAX_ROOT # where the symlink originates151MATHJAX_LIB = misc_node.MATHJAX_LIB # where the symlink points to152console.log "MATHJAX_URL = #{MATHJAX_URL}"153console.log "MATHJAX_ROOT = #{MATHJAX_ROOT}"154console.log "MATHJAX_LIB = #{MATHJAX_LIB}"155156# adds a banner to each compiled and minified source .js file157banner = new webpack.BannerPlugin(158"""\159This file is part of #{TITLE}.160It was compiled #{BUILD_DATE} at revision #{GIT_REV} and version #{SMC_VERSION}.161See #{SMC_REPO} for its #{SMC_LICENSE} code.162""")163164# webpack plugin to do the linking after it's "done"165class MathjaxVersionedSymlink166apply: (compiler) ->167# make absolute path to the mathjax lib (lives in node_module of smc-webapp)168symto = path.resolve(__dirname, "#{MATHJAX_LIB}")169console.log("mathjax symlink: pointing to #{symto}")170mksymlink = (dir, cb) ->171fs.exists dir, (exists, cb) ->172if not exists173fs.symlink(symto, dir, cb)174compiler.plugin "done", (compilation, cb) ->175async.concat([MATHJAX_ROOT, misc_node.MATHJAX_NOVERS], mksymlink, -> cb())176177mathjaxVersionedSymlink = new MathjaxVersionedSymlink()178179# deterministic hashing for assets180# TODO this sha-hash lib sometimes crashes. switch to https://github.com/erm0l0v/webpack-md5-hash and try if that works!181#WebpackSHAHash = require('webpack-sha-hash')182#webpackSHAHash = new WebpackSHAHash()183184# cleanup like "make distclean"185# otherwise, compiles create an evergrowing pile of files186CleanWebpackPlugin = require('clean-webpack-plugin')187cleanWebpackPlugin = new CleanWebpackPlugin [OUTPUT],188verbose: true189dry: false190191# assets.json file192AssetsPlugin = require('assets-webpack-plugin')193assetsPlugin = new AssetsPlugin194filename : path.join(OUTPUT, 'assets.json')195fullPath : no196prettyPrint: true197metadata:198git_ref : GIT_REV199version : SMC_VERSION200built : BUILD_DATE201timestamp : BUILD_TS202203# https://www.npmjs.com/package/html-webpack-plugin204HtmlWebpackPlugin = require('html-webpack-plugin')205# we need our own chunk sorter, because just by dependency doesn't work206# this way, we can be 100% sure207smcChunkSorter = (a, b) ->208order = ['css', 'lib', 'smc']209if order.indexOf(a.names[0]) < order.indexOf(b.names[0])210return -1211else212return 1213214# https://github.com/kangax/html-minifier#options-quick-reference215htmlMinifyOpts =216empty: true217removeComments: true218minifyJS : true219minifyCSS : true220collapseWhitespace : true221conservativeCollapse : true222223# when base_url_html is set, it is hardcoded into the index page224# it mimics the logic of the hub, where all trailing slashes are removed225# i.e. the production page has a base url of '' and smc-in-smc has '/.../...'226base_url_html = BASE_URL # do *not* modify BASE_URL, it's needed with a '/' down below227while base_url_html and base_url_html[base_url_html.length-1] == '/'228base_url_html = base_url_html.slice(0, base_url_html.length-1)229230# this is the main app.html file, which should be served without any caching231# config: https://github.com/jantimon/html-webpack-plugin#configuration232pug2app = new HtmlWebpackPlugin(233date : BUILD_DATE234title : TITLE235description : DESCRIPTION236BASE_URL : base_url_html237theme : theme238git_rev : GIT_REV239mathjax : MATHJAX_URL240filename : 'app.html'241chunksSortMode : smcChunkSorter242inject : 'body'243hash : PRODMODE244template : path.join(INPUT, 'app.pug')245minify : htmlMinifyOpts246GOOGLE_ANALYTICS : GOOGLE_ANALYTICS247)248249# static html pages250# they only depend on the css chunk251staticPages = []252# in the root directory (doc/ and policies/ is below)253for [fn_in, fn_out] in [['index.pug', 'index.html']]254staticPages.push(new HtmlWebpackPlugin(255date : BUILD_DATE256title : TITLE257description : DESCRIPTION258BASE_URL : base_url_html259theme : theme260git_rev : GIT_REV261mathjax : MATHJAX_URL262filename : fn_out263chunks : ['css']264inject : 'head'265hash : PRODMODE266template : path.join(INPUT, fn_in)267minify : htmlMinifyOpts268GOOGLE_ANALYTICS : GOOGLE_ANALYTICS269PREFIX : ''270SCHEMA : require('smc-util/schema')271PREFIX : if fn_in == 'index.pug' then '' else '../'272))273274# doc pages275for dp in (x for x in glob.sync('webapp-lib/doc/*.pug') when path.basename(x)[0] != '_')276output_fn = "doc/#{misc.change_filename_extension(path.basename(dp), 'html')}"277staticPages.push(new HtmlWebpackPlugin(278filename : output_fn279date : BUILD_DATE280title : TITLE281theme : theme282template : dp283chunks : ['css']284inject : 'head'285minify : htmlMinifyOpts286GOOGLE_ANALYTICS : GOOGLE_ANALYTICS287hash : PRODMODE288BASE_URL : base_url_html289PREFIX : '../'290))291292# the following renders the policy pages293for pp in (x for x in glob.sync('webapp-lib/policies/*.pug') when path.basename(x)[0] != '_')294output_fn = "policies/#{misc.change_filename_extension(path.basename(pp), 'html')}"295staticPages.push(new HtmlWebpackPlugin(296filename : output_fn297date : BUILD_DATE298title : TITLE299theme : theme300template : pp301chunks : ['css']302inject : 'head'303minify : htmlMinifyOpts304GOOGLE_ANALYTICS : GOOGLE_ANALYTICS305hash : PRODMODE306BASE_URL : base_url_html307PREFIX : '../'308))309310#video chat is done differently, this is kept for reference.311## video chat: not possible to render to html, while at the same time also supporting query parameters for files in the url312## maybe at some point https://github.com/webpack/webpack/issues/536 has an answer313#videoChatSide = new HtmlWebpackPlugin314# filename : "webrtc/group_chat_side.html"315# inject : 'head'316# template : 'webapp-lib/webrtc/group_chat_side.html'317# chunks : ['css']318# minify : htmlMinifyOpts319#videoChatCell = new HtmlWebpackPlugin320# filename : "webrtc/group_chat_cell.html"321# inject : 'head'322# template : 'webapp-lib/webrtc/group_chat_cell.html'323# chunks : ['css']324# minify : htmlMinifyOpts325326# global css loader configuration327cssConfig = JSON.stringify(minimize: true, discardComments: {removeAll: true}, mergeLonghand: true, sourceMap: true)328329###330# ExtractText for CSS should work, but doesn't. Also not necessary for our purposes ...331# Configuration left as a comment for future endeavours.332333# https://webpack.github.io/docs/stylesheets.html334ExtractTextPlugin = require("extract-text-webpack-plugin")335336# merge + minify of included CSS files337extractCSS = new ExtractTextPlugin("styles-[hash].css")338extractTextCss = ExtractTextPlugin.extract("style", "css?sourceMap&#{cssConfig}")339extractTextSass = ExtractTextPlugin.extract("style", "css?#{cssConfig}!sass?sourceMap&indentedSyntax")340extractTextScss = ExtractTextPlugin.extract("style", "css?#{cssConfig}!sass?sourceMap")341extractTextLess = ExtractTextPlugin.extract("style", "css?#{cssConfig}!less?sourceMap")342###343344# Custom plugin, to handle the quirky situation of extra *.html files.345# It was originally used to copy auxiliary .html files, but since there is346# no processing of the included style/js files (hashing them), it cannot be used.347# maybe it will be useful for something else in the future...348class LinkFilesIntoTargetPlugin349constructor: (@files, @target) ->350351apply: (compiler) ->352compiler.plugin "done", (comp) =>353#console.log('compilation:', _.keys(comp.compilation))354_.forEach @files, (fn) =>355if fn[0] != '/'356src = path.join(path.resolve(__dirname, INPUT), fn)357dst = path.join(@target, fn)358else359src = fn360fnrelative = fn[INPUT.length + 1 ..]361dst = path.join(@target, fnrelative)362dst = path.resolve(__dirname, dst)363console.log("hard-linking file:", src, "→", dst)364dst_dir = path.dirname(dst)365if not fs.existsSync(dst_dir)366fs.mkdir(dst_dir)367fs.linkSync(src, dst) # mysteriously, that doesn't work368369#policies = glob.sync(path.join(INPUT, 'policies', '*.html'))370#linkFilesIntoTargetPlugin = new LinkFilesToTargetPlugin(policies, OUTPUT)371372###373CopyWebpackPlugin = require('copy-webpack-plugin')374copyWebpackPlugin = new CopyWebpackPlugin []375###376377# this is like C's #ifdef for the source code. It is particularly useful in the378# source code of SMC, such that it knows about itself's version and where379# mathjax is. The version&date is shown in the hover-title in the footer (year).380setNODE_ENV = new webpack.DefinePlugin381'process.env' :382'NODE_ENV' : JSON.stringify(NODE_ENV)383'MATHJAX_URL' : JSON.stringify(MATHJAX_URL)384'SMC_VERSION' : JSON.stringify(SMC_VERSION)385'SMC_GIT_REV' : JSON.stringify(GIT_REV)386'BUILD_DATE' : JSON.stringify(BUILD_DATE)387'BUILD_TS' : JSON.stringify(BUILD_TS)388'DEBUG' : JSON.stringify(DEBUG)389390# This is not used, but maybe in the future.391# Writes a JSON file containing the main webpack-assets and their filenames.392{StatsWriterPlugin} = require("webpack-stats-plugin")393statsWriterPlugin = new StatsWriterPlugin(filename: "webpack-stats.json")394395# https://webpack.github.io/docs/shimming-modules.html396# do *not* require('jquery') but $ = window.$397# this here doesn't work, b/c some modifications/plugins simply do not work when this is set398# rather, webapp-lib.coffee defines the one and only global jquery instance!399#provideGlobals = new webpack.ProvidePlugin400# '$' : 'jquery'401# 'jQuery' : 'jquery'402# "window.jQuery" : "jquery"403# "window.$" : "jquery"404405# this is for debugging: adding it prints out a long long json of everything406# that ends up inside the chunks. that way, one knows exactly where which part did end up.407# (i.e. if require.ensure really creates chunkfiles, etc.)408class PrintChunksPlugin409apply: (compiler) ->410compiler.plugin 'compilation', (compilation, params) ->411compilation.plugin 'after-optimize-chunk-assets', (chunks) ->412console.log(chunks.map (c) ->413id: c.id414name: c.name415includes: c.modules.map (m) -> m.request416)417418419plugins = [420cleanWebpackPlugin,421#provideGlobals,422setNODE_ENV,423banner424]425426if STATICPAGES427plugins = plugins.concat(staticPages)428entries =429css : 'webapp-css.coffee'430else431# ATTN don't alter or add names here, without changing the sorting function above!432entries =433css : 'webapp-css.coffee'434lib : 'webapp-lib.coffee'435smc : 'webapp-smc.coffee'436plugins = plugins.concat([437pug2app,438#commonsChunkPlugin,439#extractCSS,440#copyWebpackPlugin441#webpackSHAHash,442#new PrintChunksPlugin(),443mathjaxVersionedSymlink,444#linkFilesIntoTargetPlugin,445])446447plugins = plugins.concat(staticPages)448plugins = plugins.concat([assetsPlugin, statsWriterPlugin])449# video chat plugins would be added here450451if PRODMODE452console.log "production mode: enabling compression"453# https://webpack.github.io/docs/list-of-plugins.html#commonschunkplugin454# plugins.push new webpack.optimize.CommonsChunkPlugin(name: "lib")455plugins.push new webpack.optimize.DedupePlugin()456plugins.push new webpack.optimize.OccurenceOrderPlugin()457# configuration for the number of chunks and their minimum size458plugins.push new webpack.optimize.LimitChunkCountPlugin(maxChunks: 5)459plugins.push new webpack.optimize.MinChunkSizePlugin(minChunkSize: 30000)460461if PRODMODE or MINIFY462# to get source maps working in production mode, one has to figure out how463# to get inSourceMap/outSourceMap working here.464plugins.push new webpack.optimize.UglifyJsPlugin465sourceMap: false466minimize: true467output:468comments: new RegExp("This file is part of #{TITLE}","g") # to keep the banner inserted above469mangle:470except : ['$super', '$', 'exports', 'require']471screw_ie8 : true472compress:473screw_ie8 : true474warnings : false475properties : true476sequences : true477dead_code : true478conditionals : true479comparisons : true480evaluate : true481booleans : true482unused : true483loops : true484hoist_funs : true485cascade : true486if_return : true487join_vars : true488drop_debugger: true489negate_iife : true490unsafe : true491side_effects : true492493494# tuning generated filenames and the configs for the aux files loader.495# FIXME this setting isn't picked up properly496if PRODMODE497hashname = '[sha256:hash:base62:33].cacheme.[ext]' # don't use base64, it's not recommended for some reason.498else499hashname = '[path][name].nocache.[ext]'500pngconfig = "name=#{hashname}&limit=16000&mimetype=image/png"501svgconfig = "name=#{hashname}&limit=16000&mimetype=image/svg+xml"502icoconfig = "name=#{hashname}&mimetype=image/x-icon"503woffconfig = "name=#{hashname}&mimetype=application/font-woff"504505# publicPath: either locally, or a CDN, see https://github.com/webpack/docs/wiki/configuration#outputpublicpath506# In order to use the CDN, copy all files from the `OUTPUT` directory over there.507# Caching: files ending in .html (like index.html or those in /policies/) and those matching '*.nocache.*' shouldn't be cached508# all others have a hash and can be cached long-term (especially when they match '*.cacheme.*')509if CDN_BASE_URL?510publicPath = CDN_BASE_URL511else512publicPath = path.join(BASE_URL, OUTPUT) + '/'513514module.exports =515cache: true516517# https://webpack.github.io/docs/configuration.html#devtool518# **do** use cheap-module-eval-source-map; it produces too large files, but who cares since we are not519# using this in production. DO NOT use 'source-map', which is VERY slow.520devtool: if SOURCE_MAP then '#cheap-module-eval-source-map'521522entry: entries523524output:525path : OUTPUT526publicPath : publicPath527filename : if PRODMODE then '[name]-[hash].cacheme.js' else '[name].nocache.js'528chunkFilename : if PRODMODE then '[id]-[hash].cacheme.js' else '[id].nocache.js'529hashFunction : 'sha256'530531module:532loaders: [533{ test: /pnotify.*\.js$/, loader: "imports?define=>false,global=>window" },534{ test: /\.cjsx$/, loaders: ['coffee-loader', 'cjsx-loader'] },535{ test: /\.coffee$/, loader: 'coffee-loader' },536{ test: /\.less$/, loaders: ["style-loader", "css-loader", "less?#{cssConfig}"]}, #loader : extractTextLess }, #537{ test: /\.scss$/, loaders: ["style-loader", "css-loader", "sass?#{cssConfig}"]}, #loader : extractTextScss }, #538{ test: /\.sass$/, loaders: ["style-loader", "css-loader", "sass?#{cssConfig}&indentedSyntax"]}, # ,loader : extractTextSass }, #539{ test: /\.json$/, loaders: ['json-loader'] },540{ test: /\.png$/, loader: "file-loader?#{pngconfig}" },541{ test: /\.ico$/, loader: "file-loader?#{icoconfig}" },542{ test: /\.svg(\?[a-z0-9\.-=]+)?$/, loader: "url-loader?#{svgconfig}" },543{ test: /\.(jpg|jpeg|gif)$/, loader: "file-loader?name=#{hashname}"},544# .html only for files in smc-webapp!545{ test: /\.html$/, include: [path.resolve(__dirname, 'smc-webapp')], loader: "raw!html-minify?conservativeCollapse"},546# { test: /\.html$/, include: [path.resolve(__dirname, 'webapp-lib')], loader: "html-loader"},547{ test: /\.hbs$/, loader: "handlebars-loader" },548{ test: /\.woff(2)?(\?[a-z0-9\.-=]+)?$/, loader: "url-loader?#{woffconfig}" },549# this is the previous file-loader config for ttf and eot fonts -- but see #1974 which for me looks like a webpack sillyness550#{ test: /\.(ttf|eot)(\?[a-z0-9\.-=]+)?$/, loader: "file-loader?name=#{hashname}" },551#{ test: /\.(ttf|eot)$/, loader: "file-loader?name=#{hashname}" },552{ test: /\.ttf(\?[a-z0-9\.-=]+)?$/, loader: "url-loader?limit=10000&mimetype=application/octet-stream" },553{ test: /\.eot(\?[a-z0-9\.-=]+)?$/, loader: "file-loader?name=#{hashname}" },554# ---555{ test: /\.css$/, loaders: ["style-loader", "css-loader?#{cssConfig}"]}, # loader: extractTextCss }, #556{ test: /\.pug$/, loader: 'pug-loader' },557]558559resolve:560# So we can require('file') instead of require('file.coffee')561extensions : ['', '.js', '.json', '.coffee', '.cjsx', '.scss', '.sass']562root : [path.resolve(__dirname),563path.resolve(__dirname, WEBAPP_LIB),564path.resolve(__dirname, 'smc-util'),565path.resolve(__dirname, 'smc-util/node_modules'),566path.resolve(__dirname, 'smc-webapp'),567path.resolve(__dirname, 'smc-webapp/node_modules')]568#alias:569# "jquery-ui": "jquery-ui/jquery-ui.js", # bind version of jquery-ui570# modules: path.join(__dirname, "node_modules") # bind to modules;571572plugins: plugins573574'html-minify-loader':575empty : true # KEEP empty attributes576cdata : true # KEEP CDATA from scripts577comments : false578removeComments : true579minifyJS : true580minifyCSS : true581collapseWhitespace : true582conservativeCollapse : true # absolutely necessary, also see above in module.loaders/.html583584585586