diff --git a/docs.json b/docs.json index acf42bcf..e4bbde04 100644 --- a/docs.json +++ b/docs.json @@ -159,7 +159,7 @@ "group": "Teams", "icon": "users", "pages": [ - "guides/teams/overview", + "guides/teams/teams-overview", "guides/teams/managing-teams", "guides/teams/teams-roles", "guides/teams/legacy-teams" @@ -588,26 +588,68 @@ "tab": "Host", "groups": [ { - "group": "Concepts", - "icon": "lightbulb", + "group": "Before You Host", + "icon": "compass", "pages": [ "host/hosting-overview", - "host/understanding-verification", - "host/earning" + "host/supported-hardware", + "host/persona-decision-guide", + "host/earning", + "host/guide-to-taxes" + ] + }, + { + "group": "Set Up", + "icon": "wrench", + "pages": [ + "host/quickstart", + "host/account-hosting-agreement", + "host/hardware-prep", + "host/storage-setup", + "host/network-ports", + "host/installing-host-software", + "host/headless-install", + "host/vms" ] }, { - "group": "Guides", - "icon": "warehouse", + "group": "Verify & List", + "icon": "badge-check", "pages": [ "host/optimization-guide", "host/verification-stages", + "host/understanding-verification", "host/how-to-self-test", "host/market-metrics", + "host/self-test-reference", + "host/pricing-your-listing", + "host/not-in-search" + ] + }, + { + "group": "Operate", + "icon": "gauge", + "pages": [ + "host/first-24-hours", + "host/reliability-uptime", + "host/maintenance-windows", + "host/removing-recreating-machines", + "host/fleet-operations", + "host/common-errors-diagnostics", + "host/machine-errors", "host/datacenter-status", "host/payment", - "host/guide-to-taxes", - "host/vms" + "host/workload-policy" + ] + }, + { + "group": "Reference", + "icon": "book-open", + "pages": [ + "host/common-host-questions", + "host/glossary", + "host/hosting-agreement", + "host/community" ] }, { @@ -808,7 +850,7 @@ "links": [ { "label": "FAQ", - "href": "/guides/reference/faq" + "href": "https://docs.vast.ai/guides/reference/faq" }, { "label": "Discord", diff --git a/host/account-hosting-agreement.mdx b/host/account-hosting-agreement.mdx new file mode 100644 index 00000000..52304469 --- /dev/null +++ b/host/account-hosting-agreement.mdx @@ -0,0 +1,50 @@ +--- +title: "Account & Hosting Agreement" +sidebarTitle: "Account & Agreement" +description: "Host account conversion and agreement flow." +"canonical": "/host/account-hosting-agreement" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist +--- + +
All host personas
+ +Use this page when a host account, hosting agreement, or Machines tab is not behaving as expected. + + +## Do I need a separate host account? + +Yes. Use a dedicated account for hosting. Do not use the same account for both client rentals and host operations, because that setup is unsupported and can cause account and machine-management issues. + +## How to accept the hosting agreement + +Once your host account is created, open the [host setup page](https://cloud.vast.ai/host/setup/). There is a link in the first paragraph to the hosting agreement. Read through the agreement. Once you accept, your account is converted to a hosting account, and a Machines link appears in the navigation. Your account can now list machines that are running the daemon software. + +The setup page is the canonical place for the current agreement link and generated install command. Do not rely on copied buttons, screenshots, or old setup instructions. + + +## What must happen before I can see host features or the Machines tab? + +Your account must be enabled for hosting and the hosting agreement must be accepted. After the account is converted to a host account, the Machines navigation and other host features should become available. + +If the account still behaves like a client account, confirm that you are signed into the intended host account and that the agreement flow completed. + + +## What if I accepted the agreement but still see a client account? + +Confirm that you are signed into the correct dedicated host account, then refresh or sign out and back in. If the Machines tab is still missing, treat it as an account-state issue rather than a Linux install issue. Contact Vast support with the host account email and screenshots or error messages from the setup flow. + +## Stale or deprecated setup links + +Copy the install command only from the official setup page while signed into the host account. Commands from old notes, screenshots, third-party guides, or another account can contain expired or account-specific material. See [Installing Host Software](/host/installing-host-software#install-command). + +## Related pages + +| Topic | Read next | +| --- | --- | +| First-time setup path | [Hosting Quickstart](/host/quickstart) | +| Installing the host software | [Installing Host Software](/host/installing-host-software) | +| Agreement reference | [Hosting Agreement](/host/hosting-agreement) | diff --git a/host/common-errors-diagnostics.mdx b/host/common-errors-diagnostics.mdx new file mode 100644 index 00000000..3b1d70d3 --- /dev/null +++ b/host/common-errors-diagnostics.mdx @@ -0,0 +1,220 @@ +--- +title: "Common Errors & Diagnostics" +sidebarTitle: "Errors & Diagnostics" +description: "Common host errors and diagnostic collection paths." +"canonical": "/host/common-errors-diagnostics" +personas: + - pro-operator + - headless-operator +--- + +
Pro OperatorHeadless / DC
+ +Use this page to collect logs and diagnose host health issues. For exact Machines-page error strings and what they usually mean, start with [Machine Error Reference](/host/machine-errors). + +## Common Dashboard Messages + +| Message | Start here | +| --- | --- | +| `Port issue` or `Port Networking Issues` | [Machine Errors: Port Networking Issues](/host/machine-errors#port-networking-issues) | +| `vericode=8` with another visible message | [Machine Error Reference](/host/machine-errors) | +| TCP works but UDP fails | [Machine Errors: TCP works but UDP fails](/host/machine-errors#tcp-works-but-udp-fails) | +| `GPU PCIE issue` or repeated AER messages | [Machine Errors: GPU PCIe Issue](/host/machine-errors#gpu-pcie-issue) | +| `failed to inject CDI devices` or `unresolvable CDI devices` | [Machine Errors: CDI Device Injection](/host/machine-errors#cdi-device-injection) | +| `bad bandwidthtest2` | [Machine Errors: bad bandwidthtest2](/host/machine-errors#bad-bandwidthtest2) | +| `nvidia-smi` fails or NVML mismatch | [`nvidia-smi` Fails](#nvidia-smi-fails) | + + +## Logs + +Installer logs are written to `vast_host_install.log` in the directory where you launched the installer, not under `/var/lib/vastai_kaalia`. + +```bash +cat vast_host_install.log +``` + +If the installer created a compressed log archive: + +```bash +tar -xzvf vastai_install_logs.tar.gz +cat vast_host_install.log +``` + +The host daemon log is: + +```bash +sudo tail -n 100 /var/lib/vastai_kaalia/kaalia.log +``` + +For self-test failures, the CLI can create a diagnostic bundle. The normal command is: + +```bash +vastai self-test machine +``` + +Failure bundles are saved by default under: + +```text +/tmp/vast_selftest__.tar.gz +``` + +You can override the output directory: + +```bash +vastai self-test machine \ + --support-bundle-dir /path/to/output +``` + +Bundles can include: + +- `self-test-output.log` +- `self-test-result.json` +- `manifest.json` +- `collection-errors.json` +- `instance/show-instance.json` +- `instance/container.log` +- `instance/daemon.log` + +For a quick SSH check: + +```bash +systemctl is-active vastai.service vast_metrics.service docker nvidia-persistenced.service +sudo journalctl -u vastai.service -n 80 --no-pager +sudo journalctl -u vast_metrics.service -n 80 --no-pager +sudo tail -n 100 /var/lib/vastai_kaalia/kaalia.log +sudo cat /var/lib/vastai_kaalia/host_port_range +``` + +Services should be `active`, logs should not show restart loops or repeated fatal errors, and the configured port range should match the forwarded ports. + + +## GPU, PCIe, And AER Kernel Logs + +Use kernel logs to distinguish a normal container/runtime error from a machine-health problem: + +- **NVRM**: NVIDIA kernel driver messages. +- **Xid**: NVIDIA GPU fault, reset, or error codes. +- **PCIe**: the bus/link between the GPU, motherboard, and CPU. +- **AER**: PCIe Advanced Error Reporting messages. Repeated AER messages can point to risers, slots, power, BIOS lane settings, cabling, motherboard, or GPU hardware issues. + +Check the current boot: + +```bash +sudo journalctl -k -b --no-pager | grep -Ei 'NVRM|Xid|AER|PCIe|fallen|GPU has fallen' +sudo dmesg -T | grep -Ei 'NVRM|Xid|AER|PCIe|fallen|GPU has fallen' +``` + +Treat repeated Xid, NVRM, PCIe, or AER errors as host hardware/driver health signals, not as isolated self-test messages. + + +## GPU PCIE Issue / AER + +`GPU PCIE issue` means the PCIe link should be investigated. Corrected AER at boot can be harmless, but repeated AER messages, uncorrectable AER, errors under load, downgraded PCIe links under load, Xid 79, or `bad bandwidthtest2` point to a real hardware path problem. + +Inspect risers, slots, cables, contacts, power, BIOS lane settings, motherboard health, and GPU health before listing again. + +Search kernel and syslog history for AER/PCIe evidence: + +```bash +sudo journalctl -k -b --no-pager | grep -Ei 'AER|PCIe Bus Error|pcieport|NVRM|Xid' +sudo journalctl -k -b -1 --no-pager | grep -Ei 'AER|PCIe Bus Error|pcieport|NVRM|Xid' +sudo dmesg -T | grep -Ei 'AER|PCIe Bus Error|pcieport|NVRM|Xid' +sudo grep -Ei 'AER|PCIe Bus Error|pcieport|NVRM|Xid' /var/log/syslog /var/log/syslog.1 /var/log/kern.log /var/log/kern.log.1 2>/dev/null +sudo zgrep -Ei 'AER|PCIe Bus Error|pcieport|NVRM|Xid' /var/log/syslog.*.gz /var/log/kern.log.*.gz 2>/dev/null +``` + +While reproducing under load, watch live kernel messages in another SSH session: + +```bash +sudo journalctl -kf | grep --line-buffered -Ei 'AER|PCIe Bus Error|pcieport|NVRM|Xid' +``` + +Correlate timestamps with self-test runs, rentals, `gpu-burn`, or other load tests. Repeated messages during load are much more concerning than a small number of corrected messages during boot. + + +## Red Machine Error + +The Machines page shows red errors detected by the host software or platform. Fix the specific cause, then allow the platform to refresh. Do not assume immediate clearing after a single fix or reboot. For exact message meanings, see [Machine Error Reference](/host/machine-errors). + + +## Port Issue / Port Networking Issues + +`Port issue`, `Port Networking Issues`, and self-test progress timeouts usually mean a direct-port reachability problem. `vericode=8` means a host-facing machine error is present; use the exact visible message to decide whether it is a port issue or another machine error. For port-networking cases, confirm the configured host port range, forward the same range to the host LAN IP, and test from outside the LAN because same-LAN tests can be misleading. + +Forward both TCP and UDP. CGNAT, double NAT without a real public forwarding path, no inbound public IP, blocked inbound ports, or incompatible IPv6-only service are not supported hosting setups. See [Network & Ports](/host/network-ports) for the full checklist. + + +## Full Client Storage + +Try cleaning Docker build cache and old unused images. For expired or deleted rentals that did not release storage, run [vastai cleanup machine](/host/cli/cleanup-machine). + + +## `nvidia-smi` Fails + +Treat this as a GPU or driver health problem. Check driver load, PCIe visibility, reboot state after updates, NVML mismatch, and NVRM/Xid/PCIe logs. Do not list a machine that cannot reliably report GPU state. + + +## NVML Driver/Library Mismatch + +This usually means the NVIDIA userspace library and loaded kernel module are different versions. Plan downtime, reboot first, then reinstall or repair the NVIDIA driver stack if needed. + + +## GPU Falls Off The PCIe Bus + +Check NVRM, Xid, and PCIe/AER logs. Power-cycle, then inspect PSU capacity, cables, risers, slots, PCIe lane settings, thermals, and any overclock/undervolt. Fix repeated bus drops before listing again. + + +## `bad bandwidthtest2` + +Treat this as a machine health or verification error. Check PCIe bandwidth, lane configuration, GPU seating, thermals, power, driver/NVML health, Xid errors, Docker GPU runtime behavior, and self-test output. Rerun self-test after fixes. + + +## Failed To Inject CDI Devices + +This points to GPU device injection or container runtime setup. First confirm the host can see every GPU and Docker can inject those GPUs into a normal container: + +```bash +nvidia-smi -L +sudo docker run --rm --gpus all nvidia/cuda:12.2.0-base-ubuntu22.04 nvidia-smi -L +``` + +If the machine is idle and not rented, you can also run a short load test with a GPU burn image you trust: + +```bash +sudo docker run --rm --gpus all oguzpastirmaci/gpu-burn 60 +``` + +If the container test fails, gather diagnostics, check Docker/NVIDIA runtime/CDI configuration, and inspect daemon/container logs. Do not assume regenerating CDI configuration fixes an unhealthy GPU, driver, or PCIe link. Escalate with logs if containers still cannot receive GPUs. + + +## Before Asking For Help + +Include: + +- Account context and exact command. +- The affected machine record in the console, relevant offer/contract context if support asks, and whether the account is the host account. +- Exact timestamps with timezone. +- Exact error strings, screenshots, or CLI output. +- What changed recently: reboot, driver, kernel, Docker, BIOS, storage, router, or ISP. +- Host OS, GPU model/count, NVIDIA driver version, and Docker storage mount output. +- Tested external IP and port for networking issues. +- Self-test support bundle when available. +- Installer log, daemon log, and relevant kernel log excerpts. + +Review bundles before sharing them. Share sensitive machine/account details only in the appropriate support channel. + + +## Escalate To Vast Support + +Escalate account conversion, hosting agreement, payout, API permission, backend machine record, ghost machine, suspected backend bug, or platform-state mismatch issues. + +Local Ubuntu, Docker, GPU driver, hardware, power, thermals, and consumer networking are primarily host responsibilities. Vast hosting requires direct public inbound TCP/UDP reachability; CGNAT or double NAT without a real public forwarding path is not a supported hosting network setup. + +## Related Pages + +| Topic | Read next | +| --- | --- | +| Install failures | [Installing Host Software](/host/installing-host-software#install-failures) | +| Machine error strings | [Machine Error Reference](/host/machine-errors) | +| Self-test errors | [Self-Test Reference](/host/self-test-reference) | +| Connectivity | [Network & Ports](/host/network-ports) | diff --git a/host/common-host-questions.mdx b/host/common-host-questions.mdx new file mode 100644 index 00000000..a27426eb --- /dev/null +++ b/host/common-host-questions.mdx @@ -0,0 +1,73 @@ +--- +title: "Common Host Questions" +sidebarTitle: "Common Questions" +description: "Common questions for Vast.ai hosts, with links to the canonical answer pages." +"canonical": "/host/common-host-questions" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist +--- + +
All host personas
+ +Use this page to route common host questions to the page that owns the answer. + +## Getting Started + +| Question | Read | +| --- | --- | +| Is Vast a good fit for me? | [Is Vast for Me?](/host/persona-decision-guide) | +| What hardware is supported? | [Supported Hardware](/host/supported-hardware) | +| What setups should I avoid? | [Unsupported setups](/host/supported-hardware#unsupported-or-discouraged-setups) | +| What is the first-time setup path? | [Hosting Quickstart](/host/quickstart) | +| Do I need a separate host account? | [Account & Hosting Agreement](/host/account-hosting-agreement#separate-host-account) | +| Why do I not see host features or Machines? | [Account & Hosting Agreement](/host/account-hosting-agreement#host-features-tab) | + +## Install And Machine Health + +| Question | Read | +| --- | --- | +| What should be ready before installing? | [Hardware Prep](/host/hardware-prep) | +| What is the Host Installer Wizard? | [Install Host Software](/host/installing-host-software#host-installer-wizard) | +| Why did install fail? | [Install failures](/host/installing-host-software#install-failures) | +| How should Docker storage be mounted? | [Storage Setup](/host/storage-setup#docker-storage-layout) | +| Is XFS project quota required? | [Storage Setup](/host/storage-setup#xfs-prjquota) | +| What Ubuntu and driver versions should I use? | [Hardware Prep](/host/hardware-prep#ubuntu-versions) | +| Where are logs and diagnostics? | [Errors & Diagnostics](/host/common-errors-diagnostics#logs) | +| What should I do for NVIDIA/NVML/CDI errors? | [Errors & Diagnostics](/host/common-errors-diagnostics) | + +## Network, Self-Test, And Verification + +| Question | Read | +| --- | --- | +| How many direct ports do I need? | [Network & Ports](/host/network-ports#ports-per-gpu) | +| Do I need TCP and UDP? | [Network & Ports](/host/network-ports#tcp-udp) | +| What if I am behind CGNAT, Starlink, IPv6-only, or no public IP? | [Network & Ports](/host/network-ports#cgnat-starlink-ipv6) | +| How do I test ports from outside my LAN? | [Network & Ports](/host/network-ports#test-ports-outside-lan) | +| What does self-test check? | [How to Self-Test](/host/how-to-self-test#what-the-self-test-checks) | +| What does `vericode=8` mean? | [Glossary](/host/glossary#vericode) and [Machine Error Reference](/host/machine-errors) | +| Why is my machine not found, not rentable, or not in search? | [Why Isn't My Machine in Search?](/host/not-in-search) | +| How does verification work? | [Understanding Verification](/host/understanding-verification) | + +## Pricing, Payouts, And Operations + +| Question | Read | +| --- | --- | +| How should I estimate earnings? | [Earnings & Pricing Model](/host/earning) | +| How should I price listings? | [Pricing Your Listing](/host/pricing-your-listing) | +| Where is market price data? | [GPU Market Metrics](/host/market-metrics) | +| When will I get paid? | [Host Payouts](/host/payment#when-paid) | +| How do I get an invoice? | [Host Payouts](/host/payment#get-an-invoice) | +| What are tax/provider rules? | [Tax Guide](/host/guide-to-taxes) | +| Why did reliability drop? | [Reliability & Uptime](/host/reliability-uptime) | +| How do I handle maintenance, cleanup, or removal? | [Maintenance Windows](/host/maintenance-windows) and [Remove or Recreate](/host/removing-recreating-machines) | + +## Support Boundary + +| Question | Read | +| --- | --- | +| What should I collect before asking for help? | [Errors & Diagnostics: collect logs](/host/common-errors-diagnostics#collect-logs) | +| What should go to Vast support? | [Errors & Diagnostics: escalate support](/host/common-errors-diagnostics#escalate-support) | +| Where can I ask other hosts? | [Discord & Community](/host/community) | diff --git a/host/community.mdx b/host/community.mdx new file mode 100644 index 00000000..5f99bc2f --- /dev/null +++ b/host/community.mdx @@ -0,0 +1,47 @@ +--- +title: "Discord & Community" +sidebarTitle: "Discord & Community" +description: "Where hosts can get community help and what to expect." +"canonical": "/host/community" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist +--- + +
All host personas
+ +Use community channels for host discussion, peer help, and machine-specific setup questions. + + +Vast does not provide hands-on machine setup support. Your OS, drivers, BIOS, storage, power, thermals, and router are host responsibilities. See [Is Vast for Me?](/host/persona-decision-guide). + + +## Joining the host Discord channels + +Join [Discord](https://discord.gg/hSuEbSQ4X8), then connect your Vast host account to unlock the **host-only channels**. + +## What to include when asking for help + +You will get faster, better answers if your first message includes: + +- The exact error string or symptom shown in the console, CLI, or self-test output. +- Machine ID and host context when safe to share in the channel. +- What you have already checked or tried. +- Relevant log excerpts or the self-test diagnostic bundle — see [what logs to collect](/host/common-errors-diagnostics#collect-logs). +- For network problems, the tested external IP:port the CLI reported. + +Review logs and bundles before posting, and never share API keys, install commands, sensitive account details, account-specific token material, or renter data in public channels. + +## What to expect + +Community help is peer support from other hosts. Vast staff may read along, but there is no response-time guarantee. For account state, payouts, backend machine records, and other platform-side issues, [escalate to support](/host/common-errors-diagnostics#escalate-support). + +## Related pages + +| Topic | Read next | +| --- | --- | +| What to escalate to Vast support | [Common Errors & Diagnostics](/host/common-errors-diagnostics#escalate-support) | +| Decoding self-test failures first | [Self-Test Reference](/host/self-test-reference) | +| Support expectations before you host | [Is Vast for Me?](/host/persona-decision-guide) | diff --git a/host/datacenter-status.mdx b/host/datacenter-status.mdx index 9b93c4df..e907ac1e 100644 --- a/host/datacenter-status.mdx +++ b/host/datacenter-status.mdx @@ -1,43 +1,41 @@ --- title: Datacenter Status +sidebarTitle: "Datacenter Status" createdAt: Mon Jan 13 2025 21:20:40 GMT+0000 (Coordinated Universal Time) updatedAt: Tue Jul 08 2025 20:21:58 GMT+0000 (Coordinated Universal Time) "canonical": "/host/datacenter-status" +personas: + - headless-operator --- +
Headless / DC
-Equipment that is in a certified datacenter on Vast.ai is eligible to be included in the "Secure Cloud" offering and receive other benefits, such as the blue datacenter label. Individual certifications will eventually be highlighted so users can understand if a given host is compliant with HIPAA, GDPR, TIER 2/3 or ISO 27001. Users typically are willing to pay more for the security and reliability that comes with equipment that is in a proper facility. - -Read through this documentation to understand the minimum requirements for becoming a datacenter partner and the specific verification steps that we will take to ensure compliance. +Datacenter status is for business-owned equipment in certified facilities. Approved machines can receive the blue datacenter label and appear in Secure Cloud searches. ## Benefits -- Blue datacenter label on all GPU offers in the web interface for equipment that is in the datacenter -- Offers are included in the "Secure Cloud" searches in the CLI and in the web interface -- Increased reliability scoring -- As a result of a few factors, generally higher search rankings in the marketplace -- Direct Discord or Slack channel to Vast.ai for support +- Blue datacenter label. +- Secure Cloud search eligibility. +- Better reliability/search signals. +- Direct Discord or Slack channel with Vast. ## Requirements -1. ONE of the following third party certificates must be active: - - ISO/IEC 27001 -2. The equipment must be owned by a business - - The business must be registered and up to date on all filings - - The owners of the business must be listed on the registration or otherwise verifiable -3. The company must sign the Datacenter Hosting Agreement -4. The owner must undergo identity verification -5. The host must have at least 5 GPU servers listed on Vast.ai or otherwise show they have a significant (5+ Servers) amount of equipment to list - -## Application Process +- Active third-party certification, currently ISO/IEC 27001. +- Business-owned equipment. +- Business registration and ownership that can be verified. +- Signed Datacenter Hosting Agreement. +- Owner identity verification. +- At least 5 GPU servers listed, or evidence of significant equipment to list. -In order to apply, you will need to first gather the required documentation: +## Apply -- Government issued IDs such as a passport for the business owner(s) -- Business information such as a certificate of good standing or other recent record -- The name and address of the datacenter where the equipment is located along with the relevant certificates -- A contract or invoice from the datacenter linking the business -- Other due dilligence documentation as required +Prepare: -Once you have the requiremed documentation, To apply please visit: [https://vast.ai/data-center-application](https://vast.ai/data-center-application) +- Government-issued ID for business owner(s). +- Business registration or certificate of good standing. +- Datacenter name, address, and certificates. +- Contract or invoice linking the business to the datacenter. +- Any additional due-diligence documents Vast requests. +Apply at [vast.ai/data-center-application](https://vast.ai/data-center-application). diff --git a/host/earning.mdx b/host/earning.mdx index 188802c4..e3c32c23 100644 --- a/host/earning.mdx +++ b/host/earning.mdx @@ -1,52 +1,127 @@ --- -title: Earning +title: "Earnings & Pricing Model" +sidebarTitle: "Earnings Model" slug: 9Ci2RCeCAEPKRT8fcy_2S createdAt: Tue Jan 14 2025 00:55:36 GMT+0000 (Coordinated Universal Time) updatedAt: Tue Aug 05 2025 19:10:27 GMT+0000 (Coordinated Universal Time) "canonical": "/host/earning" +personas: + - business-owner --- +
Business
-## Overview +Use this page to estimate gross host revenue before buying hardware, setting prices, or changing an offer. -This page in the console allows you to manage your earnings from referrals. You can find more information about Vast's referral program [here](/guides/reference/referral-program). + +Earnings are not guaranteed. Demand, supply, reliability, search filters, location, and renter requirements all affect utilization. + -# Pages Walkthrough + +## Revenue Components -![](/images/console-earning.webp) +Host earnings are based on the host-side rates you set. Client-facing totals can include separate Vast/client-side fees, so the amount a client pays can differ from the host-side earning rate shown to you. -The **Earnings **page gives you a transparent view of your referral program performance and accumulated rewards. Here’s what each section means: +Gross host revenue can include several components: -- **Current Balance:** This is the amount you’ve earned so far from your referred users but **haven’t been paid out yet. **It keeps growing as your referrals continue to use the platform. -- **Total Earnings**: This shows your **lifetime earnings **the total amount you’ve earned from all your referrals since you joined the earnings program or started hosting. It includes both paid and unpaid amounts. -- **Total Referral Count**: This number represents the **total users you’ve referred **who have successfully created accounts through your referral link. It’s a great way to track how your outreach is growing! -- **Total Rental Earnings **(host only)**: **This shows the total lifetime amount you’ve earned from your machine being rented out on the platform. -- **Total Referral Earnings** (host only): This shows the total lifetime amount you've earned from all your referrals. +```text +gross host revenue = + GPU compute revenue ++ storage revenue ++ bandwidth revenue ++ volume revenue +``` -Additionally, there is the **Earning Chart** section that provides a clear visual overview of your earning history. +| Component | What drives it | +| --- | --- | +| GPU compute | Listed GPUs rented, accepted GPU-hour rate, and running time. | +| Storage | Storage allocated over time. Storage can keep accruing while an instance is stopped or retained, depending on the instance state. | +| Bandwidth | Upload and download usage charged at the host-side bandwidth rates. | +| Volumes | Separate volume offers, if you enable and price them. | - -![](/images/console-earning-2.webp) - +Changing prices affects new rental contracts. Existing contracts use the rates accepted when the renter started the instance or reservation. -The **Template Performance** chart displays the earnings history from templates. +## Compute-Only Monthly Model -### Payouts +```text +monthly compute gross = rentable GPUs x hourly GPU price x 720 x utilization +``` -You can view your payout history for a selected date range. Here you can generate and download invoices for your earning payouts. +| Input | Meaning | +| --- | --- | +| `rentable GPUs` | GPUs expected to be rented during the month. | +| `hourly GPU price` | Expected host-side dollars per GPU-hour. Use comparable listings. | +| `720` | 30 days x 24 hours. | +| `utilization` | Share of time rented, such as `0.55` for 55%. | - -![](/images/console-earning-3.webp) - +This compute-only shortcut does not include storage, bandwidth, or volume revenue. It is also gross revenue before power, cooling, colocation, internet, hardware depreciation, repairs, taxes, and payment/accounting costs. -In the **Payout Account** section, you can set up a payout account. +## 30-Day Range - -![](/images/console-earning-4.webp) - +For planning compute revenue, compare similar machines over the last 30 days: -## Common Questions +| Scenario | Formula | +| --- | --- | +| Conservative | `GPUs x P10 price x 720 x utilization` | +| Expected | `GPUs x median price x 720 x utilization` | +| Optimistic | `GPUs x P90 price x 720 x utilization` | -### How can I have earnings as a Vast user? +Use P90 only when your machine has a real advantage: strong reliability, better network, fast storage, better location, datacenter status, or scarce GPU supply. Then add realistic storage, bandwidth, and volume assumptions separately. -You can generate earnings by gaining Vast credit through template creation via our referral program. You can find more information about Vast's referral program [here](/guides/reference/referral-program). +For a fast market-dashboard estimate, use the **GPU Overview** tab in [Host Market](https://cloud.vast.ai/host/market/): multiply `$/HR MED` by `% Rented (30D Avg)`, then multiply by `720` hours and your GPU count. See [GPU Market Metrics: GPU Overview Income Estimate](/host/market-metrics#gpu-overview-income-estimate). + +## Compute-Only Example + +A 4-GPU host at 55% utilization: + +| Price signal | Calculation | Gross/month | +| --- | --- | --- | +| P10: $0.35/hr | `4 x 0.35 x 720 x 0.55` | $554.40 | +| Median: $0.45/hr | `4 x 0.45 x 720 x 0.55` | $712.80 | +| P90: $0.65/hr | `4 x 0.65 x 720 x 0.55` | $1,029.60 | + +If actual earnings are lower than the estimate, check utilization first, then compare storage, bandwidth, and volume settings. A higher price does not help if renters choose cheaper or healthier machines. + +## What To Compare + +Compare machines with similar: + +- GPU model and count. +- Verification state and host type. +- Region, latency, and bandwidth. +- System RAM, CPU, storage, and disk size. +- Storage and bandwidth prices. +- Reliability and uptime. +- `min_gpu`, bids, discounts, and offer end date. + +## Market Data + +Start with Vast sources: + +- [GPU Market Metrics](/host/market-metrics) +- [Host market dashboard](https://cloud.vast.ai/host/market/) +- [Vast hosting earnings calculator](https://vast.ai/hosting/calculator) + +Third-party dashboards such as [500.farm Vast charts](https://500.farm/vastai/charts/) can help with comparison, but they are not the source of truth. + +## Pricing Strategy + +Start near comparable market prices, then adjust from actual utilization. + +- New machines often need competitive pricing while they build reliability history. +- Higher prices only work when renters value the machine's advantages. +- Lower prices can improve utilization, but may reduce revenue if the machine was already rented. +- Price changes affect new rental contracts only. +- Review trends over a week or month, not every short-term move. + +## Net Margin + +Subtract operating costs before judging profit: + +- Power and cooling. +- Internet, rack, or colocation. +- Hardware depreciation or financing. +- Repairs and maintenance time. +- Accounting, tax, and payment-provider costs. + +For payout timing, see [Host Payouts](/host/payment). For tax handling, see [Tax Guide for Hosts](/host/guide-to-taxes). diff --git a/host/first-24-hours.mdx b/host/first-24-hours.mdx new file mode 100644 index 00000000..57640222 --- /dev/null +++ b/host/first-24-hours.mdx @@ -0,0 +1,88 @@ +--- +title: "First 24 Hours After Install" +sidebarTitle: "First 24 Hours" +description: "Expectations after a host first installs and lists a machine." +"canonical": "/host/first-24-hours" +personas: + - hobbyist + - pro-operator +--- + +
HobbyistPro Operator
+ +Use the first day to confirm the host is stable, listed correctly, and reachable from outside your network. + + +## Healthy Daemon + +A healthy host: + +- Shows under the host account. +- Has active offers when listed. +- Has no red machine errors. +- Runs the Vast daemon, metrics service, Docker, and NVIDIA persistence service. +- Can run a temporary self-test instance that reports progress. + +SSH checks: + +```bash +systemctl is-active vastai.service vast_metrics.service docker nvidia-persistenced.service +systemctl --no-pager --full status vastai.service vast_metrics.service docker nvidia-persistenced.service +sudo journalctl -u vastai.service -n 80 --no-pager +sudo journalctl -u vast_metrics.service -n 80 --no-pager +sudo tail -n 100 /var/lib/vastai_kaalia/kaalia.log +sudo cat /var/lib/vastai_kaalia/host_port_range +``` + +The services should be `active`, without restart loops or repeated fatal errors. The configured port range should match the forwarded range. + + +## Monitor + +Watch: + +- Machine visibility and active offers. +- Self-test result. +- Red machine errors. +- Daemon, Docker, and NVIDIA runtime health. +- External port reachability. +- Thermals, power, storage, and network stability. +- First-rental startup behavior. + + +## Test Like A Client + +Use a separate client account or the CLI to rent a small test instance on your own machine. + +List visible machines/offers: + +```text +vastai show machines +``` + +Create a test instance from one available offer: + +```text +vastai create instance --image pytorch/pytorch:latest --jupyter --direct --env '-e TZ=PDT -p 22:22 -p 8080:8080' +``` + +Then test: + +- Instance appears in the client account. +- SSH command works. +- Jupyter opens. +- Ports do not hang at "connecting." + +Destroy the test instance when done. If SSH or Jupyter is stuck connecting, check [Network & Ports](/host/network-ports#test-ports-outside-lan). + +## What Can Take Time + +Verification, search visibility, and first rentals are not instant. Passing self-test makes the machine eligible; timing still depends on machine health, supply, demand, and platform policy. + +## Related Pages + +| Topic | Read next | +| --- | --- | +| Self-test | [How to Self-Test](/host/how-to-self-test) | +| Not in search | [Why Isn't My Machine in Search?](/host/not-in-search) | +| Reliability | [Reliability & Uptime](/host/reliability-uptime) | diff --git a/host/fleet-operations.mdx b/host/fleet-operations.mdx new file mode 100644 index 00000000..31285d18 --- /dev/null +++ b/host/fleet-operations.mdx @@ -0,0 +1,99 @@ +--- +title: "Fleet Operations" +sidebarTitle: "Fleet Operations" +description: "Multi-machine host workflows using the CLI and SDK: bulk listing, maintenance, monitoring, and cleanup." +"canonical": "/host/fleet-operations" +personas: + - headless-operator + - business-owner +--- + +
Headless / DCBusiness
+ +Use the CLI or SDK to manage many machines from one workstation. + +## Fleet State + +Use `--raw` for scripts and `--retry N` for unattended calls: + +```bash +vastai show machines --raw +``` + +SDK equivalents live under the [host SDK reference](/host/sdk/show-machines). + +## Bulk Listing And Pricing + +Apply listing changes to selected machines: + +```bash +vastai list machines -e 12/31/2026 --retry 6 +``` + +The same pattern works for on-demand price (`-g`), min bid (`-b`), discount rate (`-r`), min chunk (`-m`), or rolling duration (`--duration "2 weeks"`). See [vastai list machines](/host/cli/list-machines). + + +Price changes affect new rental contracts only. Extending the offer end date at the same or lower price can extend existing contracts, increasing your commitment. + + +## Maintenance Windows + +```bash +vastai schedule maint --sdate 1782950400 --duration 2 --maintenance_category power +vastai show maints --ids +vastai cancel maint +``` + +`--sdate` is Unix epoch seconds UTC. Categories are `power`, `internet`, `disk`, `gpu`, `software`, or `other`. See [Maintenance Windows](/host/maintenance-windows). + +## Default Jobs + +Run your own container when a machine is idle: + +```bash +vastai set defjob --price_gpu 0.20 --image --args +vastai remove defjob +``` + +Make sure background work complies with the [Workload Policy](/host/workload-policy). + +## Defragment GPUs + +On multi-GPU machines, defragmentation can free larger GPU groups: + +```bash +vastai defrag machines $(vastai show machines -q) +``` + +See [vastai defrag machines](/host/cli/defrag-machines). + +## Monitor + +A simple loop: + +1. Pull `vastai show machines --raw`. +2. Watch for red errors, reliability drops, missing offers, and storage pressure. +3. Inspect individual machines with [vastai show machine](/host/cli/show-machine). +4. Rerun self-test on idle machines after fixes. +5. Collect diagnostics for failures. + +For pricing and demand, use [GPU Market Metrics](/host/market-metrics). The `vastai metrics` commands also support `--raw`. + +## Cleanup And Decommissioning + +Clean stale storage: + +```bash +vastai cleanup machine +``` + +To decommission, unlist first, honor remaining rental end dates, then delete. See [Remove or Recreate](/host/removing-recreating-machines). + +## Related Pages + +| Topic | Read next | +| --- | --- | +| Headless install | [Headless Install](/host/headless-install) | +| Maintenance | [Maintenance Windows](/host/maintenance-windows) | +| Pricing | [Pricing Your Listing](/host/pricing-your-listing) | +| SDK automation | [SDK reference](/host/sdk/show-machines) | diff --git a/host/glossary.mdx b/host/glossary.mdx new file mode 100644 index 00000000..6f0a1604 --- /dev/null +++ b/host/glossary.mdx @@ -0,0 +1,119 @@ +--- +title: "Host Glossary" +sidebarTitle: "Glossary" +description: "Definitions for host-facing Vast terms, each linked to its canonical page." +"canonical": "/host/glossary" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist +--- + +
All host personas
+ +Short definitions for host-facing terms. + +### Auto Sort + +The default marketplace search order. It uses ranking factors plus randomness, so position can vary. See [Not in Search](/host/not-in-search). + +### CGNAT + +Carrier-grade NAT. Your router does not have its own public IPv4 address, so inbound connections cannot reliably reach the host. Vast hosting requires a real public inbound TCP/UDP path; CGNAT and double NAT without public forwarding are not supported hosting setups. See [Network & Ports](/host/network-ports#cgnat-starlink-ipv6). + +### Datacenter status / Secure Cloud + +A partner tier for machines in certified datacenters. Approved machines can receive the blue datacenter label. See [Datacenter Status](/host/datacenter-status). + +### Deverified + +A previously verified machine that no longer meets requirements. See [Verification Stages](/host/verification-stages). + +### Direct ports (`direct_port_count`) + +Externally reachable TCP/UDP ports forwarded to the host. Self-test requires at least 3 per listed GPU; production hosts should usually plan about 100 per listed GPU for headroom. See [Network & Ports](/host/network-ports#ports-per-gpu). + +### DLPerf + +Vast's estimated deep-learning performance score for comparing machines. See [DLPerf scoring](/guides/reference/faq/rental-types#dlperf-scoring). + +### DPH + +Dollars per hour, usually the hourly GPU compute price. See [Pricing Your Listing](/host/pricing-your-listing). + +### Instance + +The renter workload running on your machine, usually a container or VM created from an offer. + +### Interruptible rental + +Lower-priority bid rental. Higher bids run first; stopped containers and data remain on the machine. See [rental types](/guides/reference/faq/rental-types). + +### kaalia + +The Vast host daemon that runs as a systemd service. See [Errors & Diagnostics](/host/common-errors-diagnostics#logs). + +### Machines tab + +The host console page for your machines. It appears after host account conversion. See [Account & Hosting Agreement](/host/account-hosting-agreement#host-features-tab). + +### Min GPU / `min_chunk` + +The smallest GPU group a renter can select on a multi-GPU machine. See [Pricing Your Listing](/host/pricing-your-listing#listing-controls). + +### NAT hairpinning + +Router behavior that makes same-LAN public-IP port tests unreliable. Test from outside the LAN. See [Network & Ports](/host/network-ports#test-ports-outside-lan). + +### Offer + +A listing renters can accept, including price, offer end date, minimum GPU chunk, and other terms. See [Hosting Overview](/host/hosting-overview). + +### Offer end date / rental end date + +The offer end date becomes a renter's contract end date when accepted. Existing rental end dates cannot be shortened by later host changes. See [Hosting Overview](/host/hosting-overview#offer-end-date). + +### On-demand rental + +Standard highest-priority rental at the listed price. See [rental types](/guides/reference/faq/rental-types). + +### Reliability + +Vast's stability score for a machine. Reliability greater than 0.9 is an eligibility gate. See [Reliability & Uptime](/host/reliability-uptime). + +### Rental contract + +Created when a renter accepts an offer. It locks price, hardware specs, and rental end date. See [Hosting Overview](/host/hosting-overview#the-rental-contract). + +### Reserved rental + +Longer-term prepaid rental at a host-controlled discount. See [Pricing Your Listing](/host/pricing-your-listing#listing-controls). + +### Self-test + +CLI diagnostic that checks requirements, rents a temporary instance, runs runtime tests, reports results, and cleans up. See [How to Self-Test](/host/how-to-self-test). + +### Unlist vs. delete + +Unlisting stops new rentals. Deleting removes the machine record. Existing commitments must be understood first. See [Remove or Recreate](/host/removing-recreating-machines). + +### Verification / verified + +Automated status indicating a machine meets platform standards. See [Verification Stages](/host/verification-stages). + +### vericode + +A numeric verification-status signal used on offers. + +`vericode=8` means the platform has a host-facing machine `error_msg`. `Port Networking Issues` is one common `vericode=8` message, but the same bit can also appear for other machine errors such as Docker/NVIDIA runtime, CDI, PCIe, or daemon errors. + +Use the exact console message when choosing a fix. For `Port Networking Issues`, common causes include closed external ports, forwarding to the wrong LAN IP, CGNAT or double NAT without a real public forwarding path, no inbound public IP, host firewall rules, TCP/UDP mismatch, or stale port range. See [Self-Test Reference: vericode=8](/host/self-test-reference#vericode-8), [Machine Error Reference](/host/machine-errors), and [Network & Ports](/host/network-ports). + +### Volume offer + +A storage-only offer on a machine. Rented volume space reduces disk available to GPU offers. See [Hosting Overview](/host/hosting-overview#volume-offers). + +### XFS `prjquota` + +XFS project quotas used for per-container storage limits. See [Storage Setup](/host/storage-setup#xfs-prjquota). diff --git a/host/guide-to-taxes.mdx b/host/guide-to-taxes.mdx index 81b388e1..00a4cdeb 100644 --- a/host/guide-to-taxes.mdx +++ b/host/guide-to-taxes.mdx @@ -1,10 +1,15 @@ --- title: "Tax Guide for Hosts" +sidebarTitle: "Tax Guide" createdAt: "Mon Jan 13 2025 21:20:40 GMT+0000 (Coordinated Universal Time)" updatedAt: "Wed Jan 15 2025 00:36:52 GMT+0000 (Coordinated Universal Time)" canonical: "/host/guide-to-taxes" +personas: + - business-owner --- +
Business
+ **Disclaimer:** As an independent contractor, you are solely responsible for tracking your earnings and accurately reporting them in your tax filings. Vast.ai cannot provide tax advice, and we cannot verify the accuracy of any publicly available tax guidance. If you have questions about your tax obligations, please consult a tax professional. **Important:** Vast.ai does not automatically withhold taxes. @@ -35,10 +40,12 @@ If you are an international host receiving payouts via Wise, please refer to the ## Common Questions + ### Does Vast.ai handle VAT? Vast.ai is based in California and does not currently collect or remit VAT. + ### Is VAT shown on invoices? -VAT is not currently specified on Vast.ai invoices. \ No newline at end of file +VAT is not currently specified on Vast.ai invoices. diff --git a/host/hardware-prep.mdx b/host/hardware-prep.mdx new file mode 100644 index 00000000..61009951 --- /dev/null +++ b/host/hardware-prep.mdx @@ -0,0 +1,68 @@ +--- +title: "Hardware Prep" +sidebarTitle: "Hardware Prep" +description: "Ubuntu, BIOS, NVIDIA driver, and host preparation checklist." +"canonical": "/host/hardware-prep" +personas: + - pro-operator + - headless-operator +--- + +
Pro OperatorHeadless / DC
+ +Prepare the host machine before installing the Vast host software, including OS, BIOS, driver, power, thermal, and stability checks. + +## What should be ready before I run the host installer? + +Before installing, make sure the host has supported same-type GPUs, native Ubuntu/Linux, an AVX-capable CPU, at least one physical CPU core per listed GPU, adequate RAM, fast SSD/NVMe storage, working compatible GPU drivers, stable public networking, and enough direct ports forwarded for both TCP and UDP. + +For Docker storage, the preferred setup is `/var/lib/docker` mounted on XFS with project quotas enabled in `/etc/fstab`; see [Storage Setup](/host/storage-setup#fstab-example). + +For verification, the self-test preflight enforces the current minimum thresholds (CUDA support, reliability gate, direct ports, PCIe bandwidth, internet bandwidth, VRAM, system RAM). The canonical list lives in [Verification Stages: requirements](/host/verification-stages#additional-requirements-for-verification). + +For CPU capacity, use the physical-core rule: the host should have at least one visible physical CPU core per visible GPU. Hyperthreads and logical CPUs do not count as physical cores. + +## How do I inventory a clean host? + +Before changing drivers, disks, or networking, record the machine's starting state. This makes it easier to spot whether an install problem came from the provider image, the storage layout, the driver, or the Vast installer. + +```bash +lsb_release -a +uname -a +lscpu | sed -n '1,25p' +lspci | grep -i nvidia +lsblk -f +findmnt / /data0 /var/lib/docker +df -h / +ip -brief address +``` + +Save the output somewhere private and redact public IPs or account details before sharing it. If `/var/lib/docker` is already mounted, Docker already has data, or old Vast services are still installed on a supposedly clean machine, resolve that before treating the host as freshly provisioned. + + +## What Ubuntu versions should hosts use? + +Use **Ubuntu 24.04 LTS** for new installs when possible. **Ubuntu 22.04 LTS** also works and may be useful when a driver or operational constraint requires it. Confirm the OS works with your GPU driver, Docker, storage setup, and the Vast host installer before listing a production host. + +Do not blindly copy old Ubuntu 22.04 cgroup-v1 workaround commands onto Ubuntu 24.04. Docker, kernel, and cgroup defaults differ by OS version; follow the current Vast host installer path for the OS you are actually using, and treat cgroup errors as install diagnostics to fix before listing. + + +## What NVIDIA driver version should I use? + +Use a stable NVIDIA driver that supports your GPU and a CUDA runtime compatible with the Vast self-test image family. Verification requires CUDA-compatible driver/runtime support for CUDA 11.8 or newer. Verify GPU visibility and driver health with `nvidia-smi` before listing. + +## BIOS, power, and stability + +- Enable virtualization and IOMMU in BIOS if you plan to support [VMs](/host/vms); check IOMMU guidance especially on AMD EPYC systems. +- Avoid riser, lane, power, or BIOS layouts that throttle PCIe bandwidth. +- Confirm power delivery and cooling can sustain all GPUs at full load for long periods. +- Disable unattended kernel and driver updates; plan updates through [Maintenance Windows](/host/maintenance-windows#automatic-updates) instead. + +## Related pages + +| Topic | Read next | +| --- | --- | +| Buying guidance and minimums | [Supported Hardware](/host/supported-hardware) | +| Disk and filesystem layout | [Storage Setup](/host/storage-setup) | +| Ports and connectivity | [Network & Ports](/host/network-ports) | +| Installing the host software | [Installing Host Software](/host/installing-host-software) | diff --git a/host/headless-install.mdx b/host/headless-install.mdx new file mode 100644 index 00000000..75ba41ab --- /dev/null +++ b/host/headless-install.mdx @@ -0,0 +1,318 @@ +--- +title: "Headless Hosting Guide" +sidebarTitle: "Headless Install" +description: "SSH-only, Ubuntu Server install path for datacenter and remote-managed hosts." +"canonical": "/host/headless-install" +personas: + - headless-operator +--- + +
Headless / DC
+ +Use this path when the host is installed and operated entirely over SSH, BMC, or IPMI: no display, keyboard, or desktop environment. It takes a clean Ubuntu Server machine to a listed Vast host from the command line. + + +Start with a **clean install of Ubuntu 24.04.x Server** (HWE kernel recommended). During the Ubuntu installer, add OpenSSH and nothing else. Confirm the machine meets the host hardware expectations before buying, reinstalling, or listing it. + + +## First SSH Connection After Provisioning + +On a newly installed or reprovisioned host, SSH may show a new host-key fingerprint. Confirm the fingerprint against your provider console, BMC/IPMI console, or expected reprovision event before accepting it. If the host was intentionally reinstalled and the new fingerprint is correct, remove the old known-host entry and reconnect: + +```bash +ssh-keygen -R HOST_PUBLIC_IP +ssh ubuntu@HOST_PUBLIC_IP +``` + +Do not accept a changed fingerprint if you cannot explain why it changed. + +## Step 1: Update The System + +For SSH-only installs, use noninteractive `apt-get` so provider image config prompts do not block the session: + +```bash +sudo apt-get update +APT_OPTS="-o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" +sudo DEBIAN_FRONTEND=noninteractive apt-get $APT_OPTS upgrade -y +sudo DEBIAN_FRONTEND=noninteractive apt-get $APT_OPTS dist-upgrade -y +sudo DEBIAN_FRONTEND=noninteractive apt-get $APT_OPTS install -y update-manager-core +``` + +This keeps existing local config files if Ubuntu asks about files such as `/etc/issue` during an SSH upgrade. + +If you did not select the HWE kernel during install, install it now: + +```bash +APT_OPTS="-o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" +sudo DEBIAN_FRONTEND=noninteractive apt-get $APT_OPTS install --install-recommends linux-generic-hwe-24.04 -y +``` + +Reboot after kernel changes: + +```bash +sudo reboot +``` + +## Step 2: Install NVIDIA Drivers + +```bash +APT_OPTS="-o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" +sudo DEBIAN_FRONTEND=noninteractive apt-get $APT_OPTS install -y build-essential ubuntu-drivers-common +sudo apt-get update +``` + +Ask Ubuntu which NVIDIA driver it recommends for the installed GPUs: + +```bash +ubuntu-drivers devices +``` + +Install the recommended stable driver. On a clean Ubuntu 24.04 test host, Ubuntu recommended `nvidia-driver-595-open`; your machine may recommend a different version. + +```bash +APT_OPTS="-o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold" +sudo DEBIAN_FRONTEND=noninteractive apt-get $APT_OPTS install -y nvidia-driver-595-open +``` + +Reboot: + +```bash +sudo reboot +``` + +After SSH reconnects, verify: + +```bash +nvidia-smi +``` + +If `nvidia-smi` still fails after the package install, check whether `nouveau` is still bound to the GPUs and reboot before retrying the Vast installer. + +## Step 3: Disable Automatic Updates + +Automatic updates can break drivers while clients are renting your machine. Disable them and only update manually when the host is unlisted and idle: + +```bash +sudo DEBIAN_FRONTEND=noninteractive apt-get purge --auto-remove unattended-upgrades -y +sudo systemctl disable apt-daily-upgrade.timer +sudo systemctl mask apt-daily-upgrade.service +sudo systemctl disable apt-daily.timer +sudo systemctl mask apt-daily.service +``` + +Only update manually when your machine is unlisted and idle, inside a planned maintenance window. See [Maintenance Windows](/host/maintenance-windows). + +## Step 4: X Server Config + +Vast containers need the X server configuration but not a desktop environment. On Ubuntu Server, run: + +```bash +sudo nvidia-xconfig -a --cool-bits=28 --allow-empty-initial-configuration --enable-all-gpus +``` + +If the machine started life as Ubuntu Desktop, remove GNOME first: + +```bash +sudo apt-get update +sudo apt-get -y upgrade +sudo apt-get install -y libgtk-3-0 xinit xserver-xorg-core +sudo apt-get remove -y gnome-shell +sudo update-grub +sudo nvidia-xconfig -a --cool-bits=28 --allow-empty-initial-configuration --enable-all-gpus +``` + +## Step 5: Mount NVMe For Docker Storage + +This is the most important step. Vast stores client container data at `/var/lib/docker`, which should be on a dedicated drive or volume formatted as **XFS with project quotas**. See [Storage Setup](/host/storage-setup). + + +The steps below **wipe the target drive**. If your OS is on `/dev/nvme0n1`, do not use that device. Check with `lsblk` first. + + +**5a. Identify your drives:** + +```bash +lsblk -f +findmnt / +findmnt /data0 || true +findmnt /var/lib/docker || true +``` + +Find the NVMe, partition, RAID device, or LVM volume to use for Docker. It must be separate from the OS filesystem or intentionally reserved for Docker. Some bare-metal providers create a `/data0` mount; only reuse it if you have confirmed it is empty or disposable. + +**5b. Partition the drive** (replace `/dev/nvme0n1` with your device): + +```bash +sudo cfdisk /dev/nvme0n1 +``` + +In the menu: choose **gpt** if asked for a label type, select **New** (Enter for the full disk), select **Write** and type **yes**, then **Quit**. This creates a partition at `/dev/nvme0n1p1`. + +**5c. Format as XFS:** + +If the target is already mounted somewhere else, such as `/data0`, unmount it first: + +```bash +if findmnt -S /dev/nvme0n1p1 >/dev/null; then + sudo umount /dev/nvme0n1p1 +fi +``` + +```bash +sudo mkfs.xfs /dev/nvme0n1p1 +``` + +Do not use `mkfs.ext4`: Vast requires XFS for per-container storage quotas. + +**5d. Create the Docker directory:** + +```bash +sudo mkdir -p /var/lib/docker +``` + +**5e. Get the drive's UUID:** + +```bash +sudo blkid /dev/nvme0n1p1 +``` + +Copy the UUID value from the output. Using the UUID is safer than device names, which can change between reboots. + +**5f. Add to `/etc/fstab`** (replace the UUID with yours). If the same device already has an older `/data0` line, comment out or remove that old line first. + +```text +UUID=a1b2c3d4-e5f6-7890-abcd-ef1234567890 /var/lib/docker xfs rw,auto,pquota,nofail 0 0 +``` + +| Option | Purpose | +| --- | --- | +| `rw` | Read-write access. | +| `auto` | Mount automatically at boot. | +| `pquota` | Required by Vast; enables per-container storage quotas. | +| `nofail` | System still boots if the drive has an issue. | + +**5g. Mount and verify:** + +```bash +sudo mount -a +df -h /var/lib/docker +findmnt /var/lib/docker -no SOURCE,FSTYPE,OPTIONS +sudo xfs_quota -x -c "state" /var/lib/docker +``` + +You should see the NVMe or LVM volume mounted at `/var/lib/docker` with the correct size. The mount must be XFS with `pquota` or `prjquota`, and project quota accounting/enforcement must be ON. + +## Step 6: Enable GPU Persistence Mode + +GPU persistence keeps GPUs initialized between jobs so they idle at low power instead of fully reinitializing each time: + +```bash +sudo bash -c '(crontab -l 2>/dev/null; echo "@reboot nvidia-smi -pm 1" ) | crontab -' +``` + +## Step 7: Install The Vast Host Software + +Use the official Host Installer Wizard (TUI) for headless machines. Sign in to the host-enabled account, open the [Vast host setup page](https://cloud.vast.ai/host/setup/), copy the current installer command, and run it over SSH on the host machine. + +Keep the SSH session open until the wizard finishes. If the connection might drop, start a `tmux` or `screen` session first: + +```bash +tmux new -s vast-install +``` + +The wizard should validate account access, drivers, hardware, storage, ports, network speed, installation, and self-test readiness. Do not reuse old `wget`/`python3 install` commands from notes, screenshots, third-party guides, or another account. The setup-page machine installation key is not the same as a normal Vast API key. + +If the SSH terminal cannot drive the TUI reliably, use the raw installer as a headless fallback. Include both the prepared Docker storage device and the direct port range: + +```bash +read -rsp "Paste fresh Vast host setup key: " VAST_HOST_KEY +echo +sudo python3 ./install "$VAST_HOST_KEY" \ + --no-driver \ + --docker-partition /dev/mapper/vg1-lv--1 \ + --ports 40000 40799 +unset VAST_HOST_KEY +``` + +Use `--no-driver` only after `nvidia-smi` shows all GPUs. Replace the Docker storage device and ports with your actual host values. + +## Step 8: Configure GRUB Only When Needed + +Do not change kernel boot parameters during a normal Docker-only install unless you are fixing a specific driver problem or enabling [VM support](/host/vms). If you are preparing VM support, edit `/etc/default/grub` and add the VM-related options for your CPU platform: + +```text +GRUB_CMDLINE_LINUX="amd_iommu=on nvidia_drm.modeset=0" +``` + +| Option | Purpose | +| --- | --- | +| `amd_iommu=on` | Enables IOMMU on AMD CPUs; needed for [VM support](/host/vms) and PCIe passthrough. | +| `intel_iommu=on` | Use this instead of `amd_iommu=on` on Intel platforms. | +| `nvidia_drm.modeset=0` | Disables NVIDIA kernel modesetting for VM compatibility. | + +Apply the changes: + +```bash +sudo update-grub +``` + +## Step 9: Configure Networking + +Forward the direct port range from outside your network to the host for both TCP and UDP. Set the port range for client instances if you did not already pass it to the installer. This 8-GPU example gives about 100 direct ports per listed GPU: + +```bash +echo -n "40000-40799" | sudo tee /var/lib/vastai_kaalia/host_port_range +sudo systemctl restart vastai.service +sudo tail -n 50 /var/lib/vastai_kaalia/kaalia.log | grep host_port_range +``` + +Use `echo -n` so the file contains only the dash-separated range. + +Size the range for the number of GPUs you plan to list with [Network & Ports](/host/network-ports#ports-per-gpu). Test that ports are reachable **from outside your LAN**: same-LAN tests can fail from NAT hairpinning. One option is to run `sudo nc -l -p PORT` on the host and check from an external network or [portchecker.co](https://portchecker.co). + +## Step 10: Fix NVML Error If Needed + +If the installer reports an NVML error, treat it as an NVIDIA driver/library mismatch or GPU driver health issue. Plan downtime, reboot first, then reinstall or repair the NVIDIA driver stack if the mismatch remains. + +## Step 11: Reboot And Verify + +```bash +sudo reboot +``` + +After reboot, check everything is healthy: + +```bash +# Check NVMe is mounted to /var/lib/docker +df -h /var/lib/docker +findmnt /var/lib/docker -no SOURCE,FSTYPE,OPTIONS +sudo xfs_quota -x -c "state" /var/lib/docker + +# Check the Vast services and Docker are running +systemctl is-active vastai.service vast_metrics.service docker nvidia-persistenced.service + +# Check the configured port range +sudo cat /var/lib/vastai_kaalia/host_port_range + +# Check GPUs are visible +nvidia-smi +``` + +## Step 12: List, Self-Test, And Monitor + +1. List the machine and set pricing. +2. Run the self-test from any machine with the CLI: [How to Self-Test](/host/how-to-self-test). If the installer starts the self-test automatically, it may wait about 10 minutes for the daemon to initialize; watch `/var/lib/vastai_kaalia/self_test.log`. +3. If you run self-test from the same network as the host, your router must allow loopback (NAT hairpinning). Otherwise, run it from a cloud VM or a mobile connection. +4. Watch the first day over SSH: keep an eye on `vastai.service`, Docker, GPU visibility, port reachability, and the self-test log. + +## Related Pages + +| Topic | Read next | +| --- | --- | +| Host overview | [Hosting Overview](/host/hosting-overview) | +| Standard install | [Installing Host Software](/host/installing-host-software) | +| Storage layout | [Storage Setup](/host/storage-setup) | +| Port planning | [Network & Ports](/host/network-ports) | +| Self-test | [How to Self-Test](/host/how-to-self-test) | +| VM support | [VMs & IOMMU](/host/vms) | +| First day | [First 24 Hours After Install](/host/first-24-hours) | diff --git a/host/hosting-agreement.mdx b/host/hosting-agreement.mdx new file mode 100644 index 00000000..c30e4e35 --- /dev/null +++ b/host/hosting-agreement.mdx @@ -0,0 +1,50 @@ +--- +title: "Hosting Agreement" +sidebarTitle: "Hosting Agreement" +description: "Where to find the hosting agreement and what hosts commit to." +"canonical": "/host/hosting-agreement" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist +--- + +
All host personas
+ +The canonical hosting agreement lives in the console: **[cloud.vast.ai/host/agreement](https://cloud.vast.ai/host/agreement)**. You accept it when converting your account to a host account, and it governs every offer and rental contract on your machines. + + +This page is a plain-language summary for orientation only. If this page and the agreement ever differ, the [agreement](https://cloud.vast.ai/host/agreement) and the [Terms of Service](https://vast.ai/terms) win. + + +## Where to find and accept it + +| Document | Where | +| --- | --- | +| Hosting agreement (canonical) | [cloud.vast.ai/host/agreement](https://cloud.vast.ai/host/agreement) | +| Acceptance flow | Linked from the first paragraph of the [host setup guide](https://cloud.vast.ai/host/setup/) during account conversion — see [Account & Hosting Agreement](/host/account-hosting-agreement) | +| Terms of Service (platform-wide) | [vast.ai/terms](https://vast.ai/terms) | + +## What you commit to, in plain language + +By listing a machine, you create offers that clients can accept as rental contracts. For each active rental contract, you commit to: + +- Provide the hardware and services according to all the advertised specs. +- Not use the hardware for any other purpose while it is rented. +- Keep the client's data isolated and protected according to the data protection policy. +- Provide the advertised services until each rental contract's rental end date. + +For how offers become contracts and why terms lock at rental time, see [Hosting Overview: the rental contract](/host/hosting-overview#the-rental-contract). + +## Data isolation + +Renter data and workloads are off-limits: do not stop, inspect, modify, or interfere with renter containers, and do not access renter files except where a documented platform or support process requires it. The operational rules are summarized in [Workload Policy](/host/workload-policy). + +## Related pages + +| Topic | Read next | +| --- | --- | +| Accepting the agreement / account conversion | [Account & Hosting Agreement](/host/account-hosting-agreement) | +| Day-to-day policy expectations | [Workload Policy](/host/workload-policy) | +| Contract and offer lifecycle | [Hosting Overview](/host/hosting-overview) | diff --git a/host/hosting-overview.mdx b/host/hosting-overview.mdx index 5589bdf8..bfec75a0 100644 --- a/host/hosting-overview.mdx +++ b/host/hosting-overview.mdx @@ -1,56 +1,61 @@ --- title: "Hosting Overview" -slug: "xP4AEPJl2F6woPGcD7EWl" +sidebarTitle: "Hosting Overview" createdAt: "Tue Jan 14 2025 01:09:17 GMT+0000 (Coordinated Universal Time)" updatedAt: "Fri Jul 11 2025 22:44:42 GMT+0000 (Coordinated Universal Time)" "canonical": "/host/hosting-overview" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist --- +
All host personas
-Vast is a GPU marketplace. Hosts sell GPU resources on the marketplace. Hosts are responsible for: +Vast is a GPU marketplace. Hosts provide machines; renters run workloads on them. -- Setup: installing Ubuntu, creating disk partitions, installing NVIDIA drivers, opening network ports on the router and installing the Vast hosting software. -- Testing and troubleshooting all issues that can arise, such as driver conflicts, errors, bad GPUs, and bad network ports. **Vast does not offer support for getting your machine working.** Connect your Vast host account to our Discord to access our [host-only discord channels](https://discord.gg/hSuEbSQ4X8) to chat or seek help from other hosts on the platform. -- Managing your offers, including pricing and offer end dates -- Planning maintenance so that no active rental contracts are disrupted +As a host, you are responsible for: -## Account setup and hosting agreement +- Installing and maintaining Ubuntu, NVIDIA drivers, storage, networking, and the Vast host software. +- Keeping rented machines online, stable, and dedicated to renter workloads. +- Setting prices, offer end dates, and maintenance windows. +- Troubleshooting local hardware, driver, storage, and network problems. -You must create a new account for hosting. If you are using Vast.ai as a client, do not use the same account. A single client and hosting account is not supported and you will quickly run into issues. +Vast does not provide setup support for getting a machine working. For community help, connect your host account to Discord and use the [host-only channels](https://discord.gg/hSuEbSQ4X8). -Once your account is created, open the [host setup guide](https://cloud.vast.ai/host/setup/). There is a link in the first paragraph to the hosting agreement. Read through the agreement. Once you accept, your account will then be converted to a hosting account. You will notice there is now a link to Machines in the navigation, along with some other changes. Your account can now list machines that are running the daemon software. +## Start Here -## Machine setup +Create a dedicated host account from the [Vast host setup page](https://cloud.vast.ai/host/setup/). That flow links to the hosting agreement and starts host onboarding, including access to the Machines area after the agreement is accepted. -The [host setup guide](https://cloud.vast.ai/host/setup/) is the official documentation for setting up a machine on Vast.ai. Read through each section closely. +After signup, use this docs path for hardware, storage, networking, install, and self-test details. For account conversion or Machines-tab troubleshooting, see [Account & Hosting Agreement](/host/account-hosting-agreement). -Common issues to check: +For setup, follow: -- Make sure to test the networking. Clients require open ports to directly connect to the machine for most jobs. -- Make sure to read the section on IOMMU if you have an AMD EPYC system. -- Make sure to disable auto-updates so that your machine doesn't drop a client job to update a driver. +1. [Supported Hardware](/host/supported-hardware) +2. [Hardware Prep](/host/hardware-prep) +3. [Storage Setup](/host/storage-setup) +4. [Network & Ports](/host/network-ports) +5. [Installing Host Software](/host/installing-host-software) +6. [How to Self-Test](/host/how-to-self-test) -Once you are ready to list your machine, come back to this guide to understand pricing and the rental contract lifecycle. +## Host Commitment -## General concepts +Renters expect cloud-like availability. During an active rental, assume the GPUs may be used heavily and continuously. Power, cooling, internet, drivers, storage, and the host daemon must stay healthy for the full rental period. -Clients have high expectations coming from AWS or GCP. As a host, plan to offer 100% uptime for your machine during the rental period. Expect that the GPU is going to be used at close to max capacity for the rental period. Ensure that your Internet, power source and heat dissipation systems are all functioning and that you have thought through how hosting will affect each one of those items. +Do not run local gaming, mining, display workloads, or other GPU jobs on a rented machine. For policy context, see [Workload Policy](/host/workload-policy). -## Offers and Rental Contracts +## Offers And Rental Contracts -Hosts can create offers (sometimes called listings) through the CLI command list machine or the machine control panel GUI on the host machines page. +An offer is the listing renters can accept. Key offer settings include: -The main offer parameters include: +- GPU, storage, and bandwidth prices. +- Minimum GPU size (`min_gpu`). +- Interruptible minimum bid. +- Reserved discount settings. +- Offer end date. -- the pricing for GPUs,internet,storage -- the discount schedule param which determines the price difference between [on-demand](/guides/instances/rental-types) and [reserved](/guides/instances/rental-types) instances -- the min bid price for [interruptible](/guides/instances/rental-types) instances -- the min_gpu param controlling 'slicing' (explained below) -- the offer end date, which determines how long the offer accepts new rental contracts - -The offer accepts new rentals until the offer end date. When a client rents an instance on your machine, a rental contract is created from your offer. If your machine has multiple GPUs and you've set min_gpu to allow slicing, multiple clients can rent from the same offer, each creating their own independent rental contract. - -Once clients rent your machine, it is very important to honor the terms of each rental contract until its rental end date. +When a renter accepts an offer, Vast creates a rental contract. The contract locks in the offer terms at that moment: price, hardware specs, and rental end date. ![Offer, rental contract, and instance lifecycle diagram](/images/offer-contract-lifecycle.png) @@ -58,258 +63,57 @@ Once clients rent your machine, it is very important to honor the terms of each ## The Rental Contract -By listing your machine, you create an offer visible to potential clients. A rental contract is created each time a client accepts your offer by renting an instance. The rental contract locks in all of the offer's terms at the time of rental, including pricing, the rental end date, and hardware specs, and those terms cannot be changed afterward. Each rental contract is independent, you may have multiple active rental contracts from different clients on the same machine, each with its own rental end date and pricing. +Each rental contract is independent. A multi-GPU machine can have several active contracts with different renters, prices, and end dates. -As the host, you are _committing_ to provide the services as advertised in your offer: +Existing contracts cannot be changed by editing the offer. That means: -- the host must provide the hardware/services according to all the advertised specs -- the hardware can not be used for any other purposes -- the client's data must be isolated and protected according to the data protection policy -- the advertised services must be provided until each rental contract's rental end date +- Raising or lowering the offer price affects only future rentals. +- Shortening the offer end date affects only future rentals. +- Unlisting stops new rentals, but existing contracts continue. +- The machine must stay available until the latest active rental end date. -For full details, see the [hosting agreement](https://cloud.vast.ai/host/agreement). +For the full legal source, see the [Hosting Agreement](https://cloud.vast.ai/host/agreement). ### Offer End Date -The offer end date is your commitment to how long you will keep the machine online and fully functional. The pricing you set is the rate you'll be paid for that commitment. Together, these form the terms of your offer. - -You can set the offer end date in the hosting interface by clicking on the date field under expiration, or via the `end date` field in the CLI `list machine` command. Make sure to set an offer end date **before** listing your machine, or the offer will remain open indefinitely. - -#### How the offer becomes a rental contract - -When a client rents an instance from your offer, all of the offer's terms at that moment, the offer end date, the pricing, and the hardware specs, are locked into a **rental contract**. The offer end date becomes the contract's **rental end date** (shown in the UI as "client end date"), and the pricing becomes the contract's rate. Once created, a rental contract's terms cannot be changed, not by modifying the offer, not by unlisting the machine, and not by any other host action. - -#### Can I change the offer? - -Yes, but changes only affect future rental contracts. Existing rental contracts keep their original terms: - -- **Changing the price** does not change the rate on any existing rental contract -- **Shortening** the offer end date limits the commitment window for new clients, but does not shorten any existing rental end date -- **Extending** the offer end date allows new clients to rent for a longer period -- **Unlisting** the machine prevents new rental contracts entirely, but existing ones continue at their original pricing and rental end dates - -You can also unlist and relist with entirely new terms (new price, new offer end date). New rental contracts will be created at the new terms, while old rental contracts continue at their original terms. This means a single machine may have active rental contracts at different prices and different rental end dates. - -The latest rental end date across all active rental contracts on a machine is shown in the UI. You must keep the machine available until this date. - -#### Example: Multiple rental contracts from different offers - -You list a 4×A100 machine at \$2.00/GPU/hr with an offer end date of March 31. - -On January 5, Client A rents 2 GPUs. A rental contract is created at \$2.00/GPU/hr with a rental end date of March 31. +The offer end date is the latest date a new renter can accept the offer. When accepted, that date becomes the rental end date for that contract. -On January 20, you decide to raise the price. You unlist the current offer and relist at \$2.50/GPU/hr with a new offer end date of June 30. +Set an offer end date before listing. If you leave it open indefinitely, you may create a longer commitment than intended. -On February 1, Client B rents the other 2 GPUs from the new offer. A rental contract is created at \$2.50/GPU/hr with a rental end date of June 30. +### Changing An Offer -You now have two active rental contracts on the same machine at different prices and different rental end dates. Client A's contract stays at \$2.00 through March 31. Client B's contract stays at \$2.50 through June 30. You must keep the machine online and fully functional through June 30 to honor both contracts. The UI shows June 30 as the latest rental end date on this machine. +To change future terms, update the offer or unlist and relist. Existing rental contracts keep their original terms. -### Min GPU +To extend current contracts, move the offer end date later while keeping the same or lower price. If you raised the price, existing contracts will not auto-extend. -When clicking on the set pricing button, there is a min GPU field. The min GPU field allows you to set the smallest grouping of GPU rentals available on your machine in powers of 2, or down to 1. For example, if you have an 8X 3090 and set min gpu to 2, clients can create instances with 2, 4, or 8 GPUs. If you set min gpus to 1, then clients can make instances with 1, 2, 4 or 8 GPUs. +## Listing Controls -### On-demand Price +For pricing and listing strategy, use the focused pages: -The on-demand price is the price per hour for the GPU rental. On demand rentals are the highest priority and if met will stop interruptibles. - -### Interruptible min price (optional) - -The interruptible price allows for the host to set the minimum interruptible price for a client to rent. Interruptibles work in a bidding system: clients set a bid price for their instance; the current highest bid is the instance that runs, the others are paused. [more info](https://vast.ai/faq#RentalTypes) - -### Reserved Discount Pricing Factor - -Reserved Instance Discounts are a feature for clients which allows them to rent machines over a long period of time at a reduced price. The Reserved Discount Pricing Factor represents the maximum possible discount a user can achieve on your machines. - -The reserved discount pricing is determined by the hosts. If you intend to encourage a long term rental this is a factor that you may want to research. Use the filters in the UI to select reserved. - -![](/images/hosting-overview.webp) - -Once that filter is selected, hosts who offer that discount will become easily visible. Hover over the rental button to see the discount rates that are offered. The original vs. the updated price will be shown as denoted by a stikethrough in the original amount: - -![](/images/hosting-overview-2.webp) - -This discount is not static, but rather scales over time that the user rents the machine for. These values are determined by the individual host(s). - -As a host, you can set this number yourself to 0 if you wish to opt out of this feature. +| Setting | Read next | +| --- | --- | +| On-demand price | [Pricing Your Listing](/host/pricing-your-listing) | +| Interruptible minimum bid | [Pricing Your Listing](/host/pricing-your-listing#listing-controls) | +| Reserved discounts | [Pricing Your Listing](/host/pricing-your-listing#listing-controls) | +| Minimum GPU size | [Pricing Your Listing](/host/pricing-your-listing#listing-controls) | +| Earnings estimate | [Earnings & Pricing Model](/host/earning) | ## Volume Offers -In addition to GPU offers, hosts can create volume offers on machines. A volume offer is an offer for storage space, and can be priced separately from GPU offers. The space allocated for volume offers is in the same pool of space as that for GPU offers, meaning that space will not be subtracted from available offers unless it is in use. When a client rents a volume offer, they rent a subset of the total space set for the offer, up to the total amount. - -### Storage Usage - -Allocated storage (that is, storage in use by client rental contracts) is subtracted from the total storage available on a machine, and split up proportionally among the machine's GPUs in remaining GPU offers. - -For example, on a machine with 1000Gb of disk available and 2 GPUs, a host can create a volume offer of up to 1000 Gb. - -If they create a volume offer of 500 Gb, and it is not rented, the machine will be available for rent with 2 offers of 1xGPU 500Gb and 1 offer of 2xGPU 1000Gb. - -If 200 Gb of the volume offer are rented, the GPU offers will reduce to 2 1xGPU 400Gb offers and 1 2xGPU 800Gb offer. The volume offer will still remain, as there is still available space, and update to offer 300Gb. - -Similarly, if stored instances on the machine are taking up 800Gb, the volume offer will reduce to 200Gb. - -If stored instances are only taking up 400 Gb, the volume offer will not update, as there is still enough space on the machine to cover the volume offer. - -### Listing Volumes - -By default, volume offers will be listed with GPU offers at the same disk price for half of the available space on the machine. Only rented space will impact the amount of space available for GPU offers, not the space in the offer itself. You can control the amount of space listed with the -v CLI option, and the price of the space with the -z option. - -Space is listed in Gigabytes, and price in \$/Gb/Month. - -```text Text -vastai list machine -v -z -g .5 -e "12/23/2027" -r 0 -m 1 -``` - -You can also directly create a volume offer by running the `vastai list volume` command. - -```text Text -vastai list volume -s -p -``` - - - Volume offer end dates **must** align with GPU offer end dates. Setting an end date on a volume will not update if there is an existing GPU offer. Setting a GPU offer end date will update volume offer end dates. - - -Volume offers will be unlisted when the machine is unlisted. They can additionally be unlisted with the command: - -```text Text -vastai unlist volume -``` - -### Out of Sync Rental Contracts - -When a client deletes a volume, the space is automatically freed on the machine. If the machine is offline at this time, there is a job that runs hourly to free the space. If for some reason this is not working, or if you want to free the space automatically, you can run the command - -```text Text -vastai cleanup machine -``` - -This will automatically remove expired/deleted rental contracts from the machine, and available storage will update on offers. - -## Extending Rental Contracts - -To extend the current rental contracts for all clients on a given machine, change the offer end date to a later time with the same or lower pricing. - -If you have raised the pricing, you cannot extend the current rental contracts. - -## Testing your own machine - -It is vital to test your own machine to ensure the ports and software is running smoothly. - -### Setup a separate client account +Volume offers rent storage space separately from GPU offers. Rented volume space comes from the same disk pool used by GPU rentals, so it reduces storage available for GPU offers only when actually rented or occupied. -There are two supported ways to test your own machine. If you want to use the website GUI, you will need to setup a new account on a different email address, add a credit card and then find your machine and create instances on it like a client. This has the benefit of showing you the entire client experience. Testing the recommended Pytorch template is vital to ensure that SSH and Jupyter are working properly. +Volume offer end dates must align with GPU offer end dates. Unlisting the machine also unlists its volume offers. -### Use the CLI (preferred) +CLI references: -The preferred method of testing your own machine is to run the [CLI](https://cloud.vast.ai/cli/). For Windows users, we suggest setting up [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) which will require you to install Ubuntu on your Windows machine and change your bios settings to allow virtualization. Then you can start an Ubuntu terminal and run the CLI. - -To rent your own machine you will need to first search the offers with your machine ID to find the ID and then create an instance using that ID. The show machine command will show all your connected machines. - -```text Text -./vastai show machines -``` - -Then for each machine id you will need to find the available instance IDs. - -```text Text -./vastai search offers 'machine_id=12345 verified=any' -``` - -Replace 12345 with your actual machine ID in question. You can see the number of available listings as well as information about the machine. This is the fastest way to also see all the offers listed for a given machine. The website GUI stacks similar offers and so it is not easy to see all the listings for a given machine. That is not a problem for the CLI. - -Take the ID number from the first column and use that to create a free instance on your own machine. This example loads the latest pytorch image along with both jupyter and ssh direct launch modes. - -```text Text -./vastai create instance --image pytorch/pytorch:latest --jupyter --direct --env '-e TZ=PDT -p 22:22 -p 8080:8080' -``` - -You can then look at your [instance tab](https://cloud.vast.ai/instances/) to make sure that pytorch loaded correctly along with jupyter and ssh. Click on the \<\_ button to get the ssh command to connect to the instance. Test the direct ssh command. Click on the open button to test jupyter. If the button is stuck "connecting" then there is most likely a problem with the port configuration on the router in front of the machine. Once finished, destroy the instance. +- [vastai list machine](/host/cli/list-machine) +- [vastai list volume](/cli/reference/list-volume) +- [vastai unlist volume](/cli/reference/unlist-volume) ## Maintenance -The proper way to perform maintenance on your machine is to wait until all active rental contracts have ended or the machine has no running instances. - -Unlisting the offer will prevent new rental contracts from being created, but does not affect existing ones. However if you have active rental contracts, you could set the offer end date to match the latest rental end date, allowing new clients to rent instances that end at the same date. Once the end date is reached, you can then unlist the machine and then perform maintenance. - -Remember that a single machine may have multiple active rental contracts from different clients, each with its own rental end date. All rental contracts must be honored, you cannot take the machine offline until every active rental contract has ended. - -For unplanned or unscheduled maintenance, use the CLI and the schedule maint command. That will notify the client that you **have** to take the machine down and that they should save their work. You can specify a date and duration. - -## Uninstalling - -To uninstall, use the Vast uninstall script located at https://s3.amazonaws.com/vast.ai/uninstall. +Plan maintenance when the machine has no running instances or after active rental contracts end. For planned downtime, use [Maintenance Windows](/host/maintenance-windows). For unlisting, deleting, recreating, or uninstalling, see [Remove or Recreate](/host/removing-recreating-machines). ## Common Questions -### How do I host my machine(s) on Vast? How can I rent my PC? - -Hosting on Vast will require some Linux knowledge, as you will be maintaining a server. Our setup guide is [here](https://vast.ai/console/host/setup/). After the first paragraph of the guide there is a link to the hosting agreement. Once you agree, your account will be converted to a hosting account. You can review our [FAQ](https://vast.ai/faq/#Hosting-General) that answers many of your hosting questions. - -### How do I get an invoice? - -You can create an invoice by going to the "Billing" page, and then click the box for "Include Charges" under "Generate Billing History". - -### How do I check if my machine is listed? - -If your machine seems unlisted, try this command `vastai search offers 'machine_id=MACHINE_ID verified=any'` to see if the CLI finds it. If there is a result, your machine is properly listed - -### Can you verify my machine? - -Verification is conducted in a randomized and automated fashion. We only run manual verification tests are for datacenters and high end machines. - -### How does verification work? - -Verification is mostly for higher end machines, mining rigs may never be verified. Verification is also based on supply vs demand and is machine/gpu specific. Right now the only machines which can expect fast verification are \$10k+: H100 or A100 80GB - if not tested quickly in a day or so let us know. 8x4090, 4xA6000 - should be tested in less than a week, especially if you have a number of them The only manual verification tests are for datacenters and high end machines. For everything else we run more random auto verification roughly about once a week. For datacenter partner inquiries email us at [contact@vast.ai](mailto:contact@vast.ai) directly. - -### How do I gain datacenter status? - -To apply for datacenter status we have a number of requirements. There is a minimum number of servers and the datacenter where the equipment is located will need to have a third party certification such as ISO 27001. Please read the complete requirement list and application instructions [here](/host/datacenter-status). - -### How do I uninstall vast from my machine? - -You can use the [uninstall script](https://s3.amazonaws.com/vast.ai/uninstall) - -### What is this red error message on my machine? - -If the hosting software detects and error, that error message will be listed on your machine in the machines page. Once the cause of the error has been resolved, most error messages will be automatically cleared after 1-2 hours. The quickest way to learn more about resolving specific error messages is to check out the hosting channels in [our discord](https://discord.gg/hSuEbSQ4X8). - -### Why is my machine not listed? - -You won't be able to see it on the GUI right away, but you can search using the [CLI](/guides/instances/managing-instances). - -### Can I send a message to a client using my machine letting them know that I fixed an issue that they were having? - -No, there is not an established process for hosts to message clients on Vast. - -### I fear I will decrease my reliability from restarting my machine and potentially lose my verification. - -Your machine's reliability does not directly affect your verification standing. Verification is independent of reliability. Though, whenever taking your machine offline and working on it you should procede with caution as it is easy to introduce new issues or errors that will cause your machine to be de-verified. - -### How much can I make hosting on Vast? - -To get an understanding of prices, check our [Market Stats page](https://cloud.vast.ai/host/market/). You'll need to be signed in as a host with at least one registered machine to view it. - -### Why did the reliability on my machine decrease? - -If the machine loses connection or if there is a client instance that does not want to start the machine's reliability will drop. - -### How do I minimize my reliability dropping? - -Do not take your machine offline. If you must take your machine offline, minimize the time you have it offline. Note: reliability takes into account the average earnings of the machine, and machines with less earnings get penalized less from offline time. - -### If someone has already used an image on my machine does redownload happen or is the system smart? - -Prior images are cached. - -### My storage for clients is somehow full. I just have a few jobs stored in my server and most of them are old and didn't delete once the job finished. A lot of them are really old, can I remove them to free up some space? - -We suggest that you try cleaning up the docker build cache, as it sometimes frees up far more space than it claims. You can also clean up old unused images. - -### I can't find my machine? - -If your machine seems unlisted, try this command `vastai search offers 'machine_id=MACHINE_ID verified=any'` to see if the CLI finds it. If there is a result, your machine is properly listed. - -### Why can't I see my machine on the Search page in the console? - -There are over 10,000+ listings on Vast, and search only displays a small subset. You will usually not be able to find any one specific machine through most normal searches. This is expected and intentional behavior of our system. You can use `vastai search offers 'machine_id=MACHINE_ID verified=any'`, to see your machine's listing. If you want to get an understanding of the machines ranking above yours you can use very narrow filters to see what similar machines are ranking above you. For example, something like: `vastai search offers 'gpu_name=RTX_4090 cpu_ram>257 cpu_ram<258'` is a decently constrained search that will most likely include a given machine you are looking for (that fits these filters) amongst others that are similar. Keep in mind our Auto Sort that `search offers` defaults to is comprised of both ranking various factors as well as an element of randomness. \ No newline at end of file +Use [Common Host Questions](/host/common-host-questions) for quick links to setup, verification, search visibility, payouts, reliability, storage, and operations. diff --git a/host/how-to-self-test.mdx b/host/how-to-self-test.mdx index 8eed828b..e8ae81d6 100644 --- a/host/how-to-self-test.mdx +++ b/host/how-to-self-test.mdx @@ -1,120 +1,116 @@ --- title: "How to Self-Test" +sidebarTitle: "How to Self-Test" createdAt: "Mon Nov 25 2025 00:00:00 GMT+0000 (Coordinated Universal Time)" updatedAt: "Mon Nov 25 2025 00:00:00 GMT+0000 (Coordinated Universal Time)" "canonical": "/host/how-to-self-test" +personas: + - pro-operator + - headless-operator --- +
Pro OperatorHeadless / DC
-## Minimal Requirements for Verification - -Before your machine can be verified on Vast.ai, it must meet all minimum quality and reliability benchmarks. -Verification confirms that your machine is stable, performant, and properly configured to run client workloads. +Self-test checks whether a listed host is ready for verification and temporary diagnostic rentals. -**Note:** Meeting these minimum requirements makes your machine eligible for verification, but does not automatically guarantee verification. +Passing self-test makes a machine eligible for verification. It does not guarantee verification, search placement, or rentals. -## About the Self-Test - -The self-test helps confirm that your machine satisfies Vast.ai’s minimum performance and configuration standards. -It automatically evaluates key system aspects and simulates real workloads to validate both hardware and network readiness. - -## What the Self-Test Checks - -During the self-test, the following components and conditions are verified: - -* Driver setup (including CUDA configuration) -* Network speed and stability -* Open ports and connectivity -* PCIe bandwidth -* GPU VRAM capacity and reliability -* System RAM and CPU performance -* Overall machine reliability under simulated workloads - -A short test workload will be executed to assess actual runtime performance. +## Before You Run It - -**Tip:** Ensure no other jobs or instances are running during the self-test for the most accurate results. - +- Install the [Vast CLI](/cli/hello-world). +- Set the API key for the host-enabled account that owns the machine. +- List the machine. +- Make sure no client is actively renting it. +- Run from outside the host LAN when testing ports, such as a cloud VM or mobile connection. Same-LAN tests can be confused by NAT hairpinning. -## Step-by-Step Guide - -### Step 0: Install the Vast CLI - -Follow the official setup guide to install the Vast CLI: -[Vast CLI: Get Started](/cli/hello-world) - -### Step 1: Set Your API Key +```bash +vastai set api-key +``` -1.1 Get your API key: [API Keys Documentation](https://docs.vast.ai/guides/reference/keys#api-keys) + +## What Self-Test Checks -1.2 Authenticate your CLI with your Vast.ai account: +The CLI first checks listing requirements. If preflight passes, it rents a temporary diagnostic instance, starts the self-test image, polls progress, reports results, and destroys the instance. -``` -vastai set api-key -``` +Before renting the temporary diagnostic instance, the CLI checks whether the machine can be tested at all: -Example: +- The machine has a rentable offer. +- The API key can inspect the machine and offer. +- Reliability is greater than `0.90`. +- CUDA compatibility is `>= 11.8`. +- Direct ports are at least 3 ports per listed GPU. +- PCIe bandwidth is greater than 2.85 GB/s. +- Upload/download bandwidth meet the VRAM-scaled requirement. +- GPU VRAM is greater than 7 GiB per GPU. +- System RAM is close to total GPU VRAM, capped around 2 TB for very high-VRAM machines. -``` -vastai set api-key 123123123123123 -``` +Inside the temporary self-test image, checks include: -### Step 2: Run the Self-Test +- HTTPS progress endpoint reachability. +- UDP echo on the mapped UDP port. +- PyTorch/CUDA import and GPU visibility. +- Each GPU has at least 98% free VRAM. +- Enough host RAM is available. +- At least one visible physical CPU core per visible GPU. +- CUDA ResNet workload across all visible GPUs. +- GPU memory allocation, including ECC-capable memory allocation checks where applicable. +- NCCL initialization and synchronization across visible GPUs; see [`nccl_failed`](/host/self-test-reference#nccl-failed) if this stage fails. +- `stress-ng` and `gpu-burn` running together for 60 seconds. -Before running the test, make sure: +Hyperthreads and logical CPUs do not count as physical cores. -* Your machine has been listed. -* There are no active clients currently renting it. +## Run The Test -Run the self-test command: +Run: -``` +```bash vastai self-test machine ``` -Example: +To choose where failure bundles are saved: -``` -vastai self-test machine 12345 +```bash +vastai self-test machine \ + --support-bundle-dir /path/to/output ``` -### Step 3: Review the Results +Do not paste API keys or account-specific commands into public channels. -If the test passes, you’ll see: +If the test passes, the CLI reports success. -* Test completed successfully. +If it fails, look up the exact error in the [Self-Test Reference](/host/self-test-reference), fix the cause, and rerun. -If the test fails: +## Machine Not Found Or Not Rentable -* The CLI will display detailed reasons for failure. -* Apply the suggested fixes and rerun the test. +Check: -If the test says the machine is "not found or not rentable": +- The machine is listed. +- There are active offers. +- The account owns the machine. +- Reliability is above the gate unless using diagnostic mode. +- The Machines page has no red errors. +- Upload speed, download speed, RAM, and ports are populated. -* Try un-listing your machine, then listing it again. -* Ensure your machine has no missing data in your machines page, such as upload and download speed, RAM, or ports. +See [Self-Test Reference: machine not rentable](/host/self-test-reference#machine-not-rentable). -### Optional: Ignore Requirements Mode +## Fresh Install Self-Test -If your machine is rejected for not meeting requirements, but you still want to check its rentability or run the pressure tests, you can rerun the test with the `--ignore-requirements` flag: +The installer can start self-test automatically after the daemon comes online. On a fresh host, it may wait about 10 minutes before listing the machine and starting the temporary diagnostic instance. -``` -vastai self-test machine --ignore-requirements +On the host: + +```bash +sudo tail -f /var/lib/vastai_kaalia/self_test.log ``` -Example: +If reliability is still `<= 0.90`, the installer may run with `--ignore-requirements` as a diagnostic. That does not qualify the machine for verification. -``` -vastai self-test machine 12345 --ignore-requirements -``` +## Ignore Requirements Mode - -**Note:** The `--ignore-requirements` flag runs the test in a relaxed mode, bypassing some checks. -Even if the test passes in this mode, your machine does not meet the minimum verification requirements. - +Use `--ignore-requirements` only to diagnose rentability or runtime behavior. A pass in this mode does not mean the machine meets verification requirements. -**Important:** Even with `--ignore-requirements`, your machine must have at least three direct open ports, otherwise, the self-test will fail. +Even with `--ignore-requirements`, the machine still needs at least three direct open ports or self-test can fail. diff --git a/host/installing-host-software.mdx b/host/installing-host-software.mdx new file mode 100644 index 00000000..bb551a51 --- /dev/null +++ b/host/installing-host-software.mdx @@ -0,0 +1,127 @@ +--- +title: "Installing the Vast Host Software" +sidebarTitle: "Install Host Software" +description: "Canonical install guide for the Vast host installer wizard and daemon." +"canonical": "/host/installing-host-software" +personas: + - pro-operator + - headless-operator + - hobbyist +--- + +
Pro OperatorHeadless / DCHobbyist
+ +Install from the official Vast host setup flow, then confirm the daemon, Docker storage, network, and GPUs are healthy. + + + The [Vast host setup page](https://cloud.vast.ai/host/setup/) is the source of truth for the current installer command and setup flow. Use this docs page for preparation, headless guidance, log locations, and troubleshooting. + + +## Before You Install + +Complete [Hardware Prep](/host/hardware-prep), [Storage Setup](/host/storage-setup), and [Network & Ports](/host/network-ports). Your account must be host-enabled and the hosting agreement accepted; see [Account & Hosting Agreement](/host/account-hosting-agreement). + + +## Install Command + +Copy the current command from the official [Vast host setup page](https://cloud.vast.ai/host/setup/). + +The command can include an account-specific machine installation key. Do not reuse commands from old notes, shell history, screenshots, third-party guides, or another account. A regular API key is not a setup-page installation key. + + +## Host Installer Wizard + +The Host Installer Wizard is Vast's terminal UI for new host machines. + +![Host Installer Wizard terminal review screen](/images/host-installer-wizard-tui.png) + +Use the current command from the official setup page as the default install path: + +1. Sign in to the host-enabled account. +2. Open the [host setup page](https://cloud.vast.ai/host/setup/). +3. Copy the current installer command. +4. Run it on the host, locally or over SSH. +5. Fix failed checks before continuing. + +The wizard checks account access, OS/GPU readiness, storage, network ports, network speed, install progress, and self-test readiness. + + +For SSH installs, keep the session open or use `tmux`. + + +If the wizard stops at NVIDIA/CUDA checks, repair the driver, reboot, confirm `nvidia-smi` shows every GPU, then rerun the wizard. + + +## Headless Fallback + +Use the TUI when possible. If the SSH terminal cannot drive it reliably, use the raw installer only after the host is already prepared: + +- `nvidia-smi` shows every GPU. +- `/var/lib/docker` is XFS with project quotas. +- TCP and UDP are forwarded for the direct port range. +- You copied a fresh setup-page machine installation key. + +Example: + +```bash +mkdir -p ~/vast-install +cd ~/vast-install +wget https://console.vast.ai/install -O install +read -rsp "Paste fresh Vast host setup key: " VAST_HOST_KEY +echo +sudo python3 ./install "$VAST_HOST_KEY" \ + --no-driver \ + --docker-partition /dev/mapper/vg1-lv--1 \ + --ports 40000 40799 +unset VAST_HOST_KEY +``` + +Replace the device and port range with your host values. Use `--no-driver` only after verifying the NVIDIA driver yourself. Do not reset machine identity unless Vast support tells you to. + + +Keep setup keys out of screenshots, tickets, and shell history. + + + +## Install Failures + +Common causes: + +- Stale setup command. +- Account not fully host-enabled. +- Interrupted installer download. +- Driver or Docker/cgroup problem. +- Blocked ports or low upload speed. +- Storage quota/filesystem mismatch. + +Refresh the setup page, copy a fresh command, confirm host account access, and rerun the wizard. Use the wizard error, installer log, and daemon logs to decide what to fix. + +## Logs + +Check `vast_host_install.log`, the `vastai.service` systemd unit, and the kaalia logs. For log locations and diagnostic bundles, see [Common Errors & Diagnostics](/host/common-errors-diagnostics#logs). + +## After Install + +Confirm the machine appears under your host account, then check: + +```bash +systemctl is-active vastai.service vast_metrics.service docker nvidia-persistenced.service +findmnt /var/lib/docker -no SOURCE,FSTYPE,OPTIONS +sudo xfs_quota -x -c "state" /var/lib/docker +sudo cat /var/lib/vastai_kaalia/host_port_range +``` + +Expected: + +- Services report `active`. +- `/var/lib/docker` is XFS with `pquota` or `prjquota`. +- XFS project quota accounting and enforcement are ON. +- Port range matches your firewall/router forwarding. + +The installer may start self-test automatically. On a fresh host it can wait about 10 minutes for daemon initialization: + +```bash +sudo tail -f /var/lib/vastai_kaalia/self_test.log +``` + +Then continue with [How to Self-Test](/host/how-to-self-test) and [First 24 Hours](/host/first-24-hours). diff --git a/host/machine-errors.mdx b/host/machine-errors.mdx new file mode 100644 index 00000000..928d55ae --- /dev/null +++ b/host/machine-errors.mdx @@ -0,0 +1,338 @@ +--- +title: "Machine Error Reference" +sidebarTitle: "Machine Errors" +description: "Host Machines page error strings, meanings, and next checks." +"canonical": "/host/machine-errors" +personas: + - pro-operator + - headless-operator +--- + +
Pro OperatorHeadless / DC
+ +Use this page when the Machines page shows a red machine error or a host-facing runtime message. Start with the exact text shown in the console, fix the likely host-side cause, then allow the platform to refresh the machine state. + + +Do not reset machine identity, force-refresh machine specs with internal scripts, or delete renter data as a first response. If the page state looks wrong after the host is healthy, collect logs and escalate to Vast support. + + +## What Machine Errors Mean + +Machine errors usually mean the host software or platform detected a problem with networking, Docker, NVIDIA runtime, GPU health, storage, VM support, or self-test runtime behavior. + +Errors have different impact levels. Some de-verify the whole machine and take it off the market until the error clears. Some only disable VM GPU-passthrough offers while container rentals can remain listed. Some are logged against a single rental attempt and do not change the machine's listing state by themselves. + +Resolved errors may take time to clear because the host daemon and platform need to observe healthy state again. Do not assume immediate clearing after a single fix or reboot. + +## Listing Impact + +| Impact | What happens | Examples | What to do | +| --- | --- | --- | --- | +| Machine de-verified | The machine is marked unverified and taken off the market until the error clears. | Port networking issues, Docker/NVIDIA runtime errors, GPU PCIe/ECC faults, Docker daemon unavailable, XFS project quota errors | Fix the host-side cause, verify the host is healthy, then allow the platform to refresh. | +| VM offers disabled | The machine can stay listed for container rentals, but VM GPU-passthrough rentals are disabled while the VM condition persists. | IOMMU changes, GDM holding a GPU, VM memory preflight failure, machine authentication denied while fetching VM secrets | Fix the VM-specific prerequisite, then wait for the machine to report clean state again. | +| Logged only | The message is recorded against a specific rental attempt and does not de-verify or unlist the machine by itself. | Container create, build, volume, template-update, or secrets-service messages | Use the message as support context if the rental failed or repeats. | + +## Quick Lookup + +| Message or symptom | Likely area | Start here | +| --- | --- | --- | +| `Port issue`, `port networking issues`, `Port Networking Issues` | Public port reachability | [Port Networking Issues](#port-networking-issues) | +| `vericode=8` with another visible message | Host-facing machine `error_msg` | Use the exact message in this table | +| TCP works but UDP fails | Public port protocol mismatch | [TCP works but UDP fails](#tcp-works-but-udp-fails) | +| `nvidia-container-cli: device error:` | NVIDIA container runtime or GPU device state | [NVIDIA container runtime](#nvidia-container-runtime) | +| `failed to inject CDI devices: unresolvable CDI devices` | NVIDIA CDI / Docker GPU injection | [CDI device injection](#cdi-device-injection) | +| `unknown or invalid runtime nvidia` | NVIDIA container runtime not available to Docker | [Docker runtime configuration](#docker-runtime-configuration) | +| `unknown flag: runtime` | Docker CLI/runtime invocation problem | [Docker runtime configuration](#docker-runtime-configuration) | +| `Cannot connect to the Docker daemon` | Docker daemon unavailable | [Docker daemon unavailable](#docker-daemon-unavailable) | +| `GPU PCIE issue`, AER, PCIe, Xid 79, fallen off bus | Hardware path / PCIe / power / riser | [GPU PCIe issue](#gpu-pcie-issue) | +| `bad bandwidthtest2` | GPU transfer or PCIe bandwidth health | [`bad bandwidthtest2`](#bad-bandwidthtest2) | +| `Unable to determine the device handle for` | GPU disappeared or cannot be enumerated | [GPU missing or unhealthy](#gpu-missing-or-unhealthy) | +| `CUDA error: uncorrectable ECC error encountered` | GPU memory / ECC fault | [ECC and GPU memory errors](#ecc-and-gpu-memory-errors) | +| `CUDA is not available` | Driver/runtime/container CUDA visibility | [CUDA unavailable](#cuda-unavailable) | +| `nccl_failed`, NCCL initialization, sync, or communicator error | Multi-GPU communication | [NCCL failed](#nccl-failed) | +| `--storage-opt is supported only for overlay over xfs` | Docker storage filesystem/quota | [XFS project quota error](#xfs-project-quota-error) | +| Full client storage, `no space left on device` | Docker storage capacity | [Full client storage](#full-client-storage) | +| Machine not found or not rentable | Offer/listing/rentability state | [Machine not rentable](#machine-not-rentable) | +| Red machine error / unhealthy / deverified | Machine health or platform state | [Generic red machine error](#generic-red-machine-error) | +| `` | Manual Vast admin review flag | [Admin-set machine error](#admin-set-machine-error) | +| IOMMU, GDM, machine does not support VMs, VM memory preflight, or VM secrets HTTP 401 | VM host support | [VM offer errors](#vm-offer-errors) | +| Container create, build, volume, template update, or secrets-service message | Single rental attempt | [Rental-attempt messages](#rental-attempt-messages) | + + +## Port Networking Issues + +`Port issue`, `port networking issues`, and `Port Networking Issues` usually mean Vast could not reach the machine through the configured direct ports. `vericode=8` by itself only tells you a host-facing machine `error_msg` exists; use the exact visible message to decide whether it is a port issue, CDI issue, PCIe issue, daemon issue, or another machine error. + +Check: + +- The host port range configured in the installer or `/var/lib/vastai_kaalia/host_port_range`. +- Router, firewall, or datacenter ACL forwarding for the same range. +- Public inbound IP availability. +- Both TCP and UDP forwarding. +- Outside-LAN reachability. Same-LAN tests can pass because of NAT hairpinning. + +If a self-test failure prints a public IP and port, troubleshoot that exact endpoint. See [Network & Ports](/host/network-ports) and [Self-Test Reference: vericode=8](/host/self-test-reference#vericode-8). + + +## TCP Works But UDP Fails + +TCP-only forwarding can make web or progress checks pass while UDP workloads fail. The direct port range must forward both TCP and UDP. + +A UDP failure can still happen when the UDP port appears mapped and the container-side responder starts. The public UDP path still has to work end to end. The paid dogfood run on 2026-06-24 reproduced this shape: the TCP progress endpoint was reachable, but the external UDP echo probe timed out. Treat this as a network-path issue first, not a CUDA or NCCL failure. + +Check router/firewall rules, datacenter ACLs, host firewall rules, and whether the provider blocks UDP. Test from outside the LAN with `tcpdump` on the host and a UDP packet from an external machine. See [Network & Ports: TCP and UDP](/host/network-ports#tcp-udp). + + +## NVIDIA Container Runtime + +`nvidia-container-cli: device error:` means Docker attempted to start a GPU container, but the NVIDIA runtime could not expose one or more GPU devices. + +Possible causes include unhealthy or missing GPUs, driver/NVML issues, CDI/runtime configuration, PCIe errors, or a stale device state after a GPU reset. + +Start with: + +```bash +nvidia-smi -L +sudo docker run --rm --gpus all nvidia/cuda:12.2.0-base-ubuntu22.04 nvidia-smi -L +sudo journalctl -k -b --no-pager | grep -Ei 'NVRM|Xid|AER|PCIe|fallen' +``` + +If Docker cannot inject GPUs into a normal container, fix driver/runtime/GPU health before listing again. See [Errors & Diagnostics: Failed To Inject CDI Devices](/host/common-errors-diagnostics#failed-cdi). + + +## CDI Device Injection + +`failed to inject CDI devices: unresolvable CDI devices` points to GPU device injection or container runtime setup. It can also be a symptom of a GPU that disappeared from the host after a reset or PCIe fault. + +Confirm host GPU visibility and Docker GPU injection: + +```bash +nvidia-smi -L +sudo docker run --rm --gpus all nvidia/cuda:12.2.0-base-ubuntu22.04 nvidia-smi -L +``` + +If both fail, investigate driver, NVML, PCIe, Xid, and Docker/NVIDIA runtime health. Do not assume a CDI regeneration command fixes the underlying issue. + + +## Docker Runtime Configuration + +`unknown or invalid runtime nvidia` usually means Docker cannot use the NVIDIA runtime. + +`unknown flag: runtime` means the Docker command rejected the `--runtime` option. This can happen if the command is being handled by a nonstandard wrapper or incompatible Docker path. + +Check: + +```bash +type docker +docker --version +docker create --help | grep -- --runtime +docker info | grep -i runtime +``` + +Then confirm a normal GPU container can run. Repair Docker/NVIDIA runtime configuration before accepting rentals. + + +## Docker Daemon Unavailable + +`Cannot connect to the Docker daemon` means the host software could not talk to Docker. + +Check: + +```bash +systemctl is-active docker +sudo journalctl -u docker -n 100 --no-pager +sudo docker ps +``` + +Fix Docker startup, storage, or daemon configuration errors before rerunning self-test or listing the machine. + + +## GPU PCIe Issue + +`GPU PCIE issue`, repeated AER messages, Xid 79, or “GPU has fallen off the bus” point to the hardware path between the GPU and the host. + +Check risers, slots, cabling, PSU capacity, thermals, BIOS lane settings, motherboard health, GPU seating, and overclock/undervolt settings. Review kernel logs under load: + +```bash +sudo journalctl -k -b --no-pager | grep -Ei 'AER|PCIe|NVRM|Xid|fallen' +sudo dmesg -T | grep -Ei 'AER|PCIe|NVRM|Xid|fallen' +``` + +Corrected AER at boot can be harmless, but repeated messages during load are a stronger signal. See [Errors & Diagnostics: GPU PCIE Issue](/host/common-errors-diagnostics#gpu-pcie-issue). + + +## `bad bandwidthtest2` + +`bad bandwidthtest2` means a GPU transfer or bandwidth health check failed. Treat it as a machine-health or verification error. + +Use the reported GPU index if present, then check PCIe bandwidth, lane configuration, GPU seating, risers, thermals, power, driver/NVML health, Xid errors, Docker GPU runtime behavior, and self-test output. + + +## GPU Missing Or Unhealthy + +`Unable to determine the device handle for` usually means NVIDIA tooling could not enumerate a GPU that should exist. + +Check whether the GPU is visible to the PCIe bus and NVIDIA driver: + +```bash +lspci | grep -i nvidia +nvidia-smi -L +sudo journalctl -k -b --no-pager | grep -Ei 'NVRM|Xid|fallen|AER|PCIe' +``` + +If a GPU has fallen off the bus, power-cycle the host and fix the hardware path before listing again. + + +## ECC And GPU Memory Errors + +`CUDA error: uncorrectable ECC error encountered` means CUDA reported an uncorrectable GPU memory error. This is a hardware-health signal, not a pricing or marketplace issue. + +Check ECC and remapping state: + +```bash +nvidia-smi -q -d ECC +nvidia-smi -q | grep -iE 'Xid|Remapped|Pending' +sudo journalctl -k -b --no-pager | grep -i xid +``` + +If the error repeats, remove the machine from active hosting until the GPU is repaired or replaced. + + +## CUDA Unavailable + +`CUDA is not available` means the container or self-test image could not use the GPU through CUDA. + +Check that `nvidia-smi` works on the host, Docker GPU injection works, the NVIDIA driver supports the CUDA runtime used by the image, and no GPU is missing or unhealthy. Verification requires CUDA-compatible driver/runtime support for CUDA 11.8 or newer. + + +## NCCL Failed + +`nccl_failed` or NCCL initialization/synchronization errors mean multi-GPU communication failed after the test started. + +First confirm basic GPU visibility and container GPU access. Then check CUDA/NCCL/framework compatibility, PCIe/NVLink/P2P topology, shared memory, overloaded or unhealthy GPUs, and network/IB configuration if present. + +On HGX, DGX, or other NVSwitch systems, also check Fabric Manager/NVLSM on the host. The self-test container can show NCCL symptoms, but it cannot prove host service state: + +```bash +systemctl status nvidia-fabricmanager +journalctl -u nvidia-fabricmanager --since "-24h" +nvidia-smi -q | grep -i -A 2 Fabric +nvidia-smi topo -m +``` + +See [Self-Test Reference: nccl_failed](/host/self-test-reference#nccl-failed). + + +## XFS Project Quota Error + +`--storage-opt is supported only for overlay over xfs` means Docker storage is not on the expected XFS mount with project quotas. + +Vast uses Docker storage quotas for renter containers. Do not bypass the error by removing Docker quota settings. Move or remount `/var/lib/docker` onto the intended XFS filesystem, enable `pquota` or `prjquota` in `/etc/fstab`, verify quota state, then restart Docker and host software. + +See [Storage Setup: XFS Project Quotas](/host/storage-setup#xfs-prjquota). + + +## Full Client Storage + +Full client storage, `no space left on device`, or missing expected disk space usually means Docker storage is full, mounted on the wrong disk, or still holding images, stopped containers, volumes, or expired rental data. + +Check: + +```bash +df -h /var/lib/docker +sudo docker system df +findmnt /var/lib/docker -no SOURCE,FSTYPE,OPTIONS +``` + +Do not manually delete renter data. For expired or deleted rentals that did not release storage, use [vastai cleanup machine](/host/cli/cleanup-machine). + + +## Machine Not Rentable + +“Machine not found or not rentable” can mean the machine is offline, unlisted, already rented, missing active offers, below a threshold, showing a red machine error, missing required details, or not visible to the account running the command. + +Start in the Machines page for the host account. Confirm the machine is listed, online, not currently rented in a way that hides the expected offer, has active offers, and shows required details such as ports, RAM, upload speed, and download speed. Then confirm the CLI is authenticated to the same host-enabled account. + +See [Self-Test Reference: machine not rentable](/host/self-test-reference#machine-not-rentable) and [Not in Search](/host/not-in-search). + + +## Generic Red Machine Error + +A red machine error means the host software or platform detected a machine-health, network, storage, driver, container, VM, or policy issue. Fix the specific cause first, then allow the platform to refresh. + +If the host looks healthy but the console state remains wrong, collect logs and escalate to Vast support. Do not assume support can manually verify ordinary machines; verification and most eligibility checks are automated. + + +## Admin-Set Machine Error + +`` means a Vast administrator manually flagged the machine for investigation. This is not an automatic host self-test result. + +Do not try to clear this by resetting machine identity or running internal scripts. Review the visible console message, collect host logs and recent-change details, then follow up with Vast support. + + + +## VM Offer Errors + +VM-related messages can disable VM GPU-passthrough offers while leaving ordinary container rentals listed. These errors clear after the underlying condition is fixed and the machine reports clean state again, but exact timing can vary. + +| Message | Likely cause | Start here | +| --- | --- | --- | +| `IOMMU config or Nvidia DRM Modeset has changed to no longer support VMs` | IOMMU or passthrough setup changed, a GPU shares an incompatible IOMMU group, or NVIDIA DRM modeset became enabled. | Check BIOS IOMMU/VT-d/AMD-Vi, GPU isolation, and NVIDIA modeset settings. | +| `GDM is on; VMs will no longer work.` | GDM is running and holding a GPU, which conflicts with VFIO passthrough. | Disable the display manager for VM-capable headless hosting, then reboot or re-check GPU ownership. | +| `Error: machine does not support VMs.` | A VM failed because hardware virtualization or IOMMU is not enabled or available. | Enable Intel VT-d or AMD-Vi/IOMMU in BIOS if the hardware supports it. | +| `Error: Unexpected configuration change; cannot assign GPUs to VMs.` | GPU IOMMU grouping changed since verification. | Review BIOS, kernel, driver, and GPU-slot changes made after listing. | +| `Error: Machine incompatible with VMs after host change. Please destroy the instance and find a new machine.` | Host configuration changed and the machine no longer meets VM-passthrough prerequisites. | Treat this as VM-specific incompatibility; do not reset machine identity to work around it. | +| `Error: unable to complete out-of-memory-check` | VM preflight could not confirm enough usable host memory. | Check host RAM pressure, reserved memory, and whether too many workloads are active. | +| `Secrets fetch failed: machine authentication denied (HTTP 401)` | Machine credentials were rejected while fetching instance secrets. | Collect logs and escalate if it repeats; do not use machine-ID reset workflows. | +| `Error: GPU error, unable to start instance.` | A GPU error blocked VM startup. Repeated occurrences can disable VM offers. | Check GPU health, driver/NVML state, Xid logs, and VM prerequisites. | + +See [VMs & IOMMU](/host/vms). + + +## Rental-Attempt Messages + +Some messages are logged against a specific rental attempt and do not change the machine's verification or listing by themselves. They can still explain why a specific rental failed. + +| Message | Meaning | +| --- | --- | +| `Error: Internal error.` | Unexpected internal error while creating the container. | +| `Error: container is mounted` | A leftover mount from a previous run blocked the operation. | +| `Error: Requested volume does not exist: ` | The requested storage volume was deleted or never existed. | +| `docker_build(): ` | Docker build failed and wrapped the build's own output. | +| `Waiting for template update` | Transitional state while the image or template updates. This is not a machine fault by itself. | +| `unknown` | The failed operation did not provide a more specific message. | +| `Secrets fetch failed: network/subprocess error` | The secrets service could not be reached because of network or local subprocess failure before a response. | +| `Secrets fetch failed: empty response` | The secrets service returned an empty body. | +| `Secrets fetch failed: instance not found or access denied (HTTP 404)` | The instance was not found, or access was denied. | +| `Secrets fetch failed: bad request (HTTP 400)` | The secrets request was malformed. | +| `Secrets fetch rate limited (HTTP 429)` | The secrets service rate-limited the request. | +| `Secrets fetch failed: server error (HTTP )` | The secrets service returned a 5xx server error. | +| `Secrets fetch failed: unexpected HTTP ` | The secrets service returned another non-success status. | +| `Secrets fetch JSON parse error` | The secrets response was not valid JSON. | + +If one of these repeats across rentals, collect the exact message, timestamp, instance details, and host logs before asking for help. + + +## What To Collect + +Before asking for help, collect: + +- Exact error string and screenshot. +- The affected machine record in the console and whether you are using the host account. +- Timestamp with timezone. +- Recent changes: reboot, driver, kernel, Docker, BIOS, storage, router, ISP, or GPU swap. +- `nvidia-smi -L`, Docker GPU container test output, and relevant kernel logs. +- Host daemon logs and installer logs if this started during install. +- Self-test support bundle if the error came from self-test. + +Review bundles before sharing them. Share sensitive account or machine details only in the appropriate support channel. + +## Related Pages + +| Topic | Read next | +| --- | --- | +| General diagnostics and logs | [Errors & Diagnostics](/host/common-errors-diagnostics) | +| Self-test failures | [Self-Test Reference](/host/self-test-reference) | +| Port forwarding | [Network & Ports](/host/network-ports) | +| Docker storage | [Storage Setup](/host/storage-setup) | +| Search and visibility | [Not in Search](/host/not-in-search) | diff --git a/host/maintenance-windows.mdx b/host/maintenance-windows.mdx new file mode 100644 index 00000000..d667373c --- /dev/null +++ b/host/maintenance-windows.mdx @@ -0,0 +1,100 @@ +--- +title: "Maintenance Windows" +sidebarTitle: "Maintenance Windows" +description: "How hosts plan maintenance without disrupting active contracts." +"canonical": "/host/maintenance-windows" +personas: + - pro-operator + - headless-operator +--- + +
Pro OperatorHeadless / DC
+ +Use maintenance windows to plan host work without surprising active renters. + + +## Before Maintenance + +Wait until active rental contracts have ended, or until the machine has no running instances, before planned maintenance. Unlisting prevents new rental contracts, but it does not end existing contracts. + +Check the machine before you change its listing state: + +```bash +vastai show machines +vastai show machine +``` + + + A single machine can have multiple active rental contracts from different clients. Do not take the machine offline until every active rental contract has ended. + + + +## Planned Maintenance + +For planned work, set the offer end date to the maintenance date so new rentals do not run past your planned downtime. Existing contracts keep their accepted end date, so choose a date that still honors the latest active rental. + +```bash +vastai list machine --end_date MM/DD/YYYY +``` + +For multiple machines, use the fleet command: + +```bash +vastai list machines --end_date MM/DD/YYYY --retry 6 +``` + +When the active contracts have ended, unlist the machine before taking it down: + +```bash +vastai unlist machine +``` + +See [vastai list machine](/host/cli/list-machine), [vastai list machines](/host/cli/list-machines), and [vastai unlist machine](/host/cli/unlist-machine). + + +## Schedule An Unplanned Window + +If you must take the machine down unexpectedly, schedule a maintenance window so clients are notified and can save their work. The start date is Unix epoch seconds in UTC, and the duration is in hours. + +```bash +vastai schedule maint --sdate --duration --maintenance_category software +``` + +Example: + +```bash +vastai schedule maint 8207 --sdate 1782950400 --duration 2 --maintenance_category power +``` + +Maintenance categories are `power`, `internet`, `disk`, `gpu`, `software`, or `other`. See [vastai schedule maint](/host/cli/schedule-maint). + + +## Check Or Cancel Maintenance + +Check scheduled maintenance for one or more machines: + +```bash +vastai show maints --ids +vastai show maints --ids , +``` + +Cancel scheduled maintenance for a machine: + +```bash +vastai cancel maint +``` + +See [vastai show maints](/host/cli/show-maints) and [vastai cancel maint](/host/cli/cancel-maint). + + +## Should I disable automatic updates? + +Avoid unattended kernel or GPU driver changes during active rentals. Driver and kernel updates can break running jobs or leave the host in a driver/library mismatch state until rebooted. Plan updates during maintenance windows, then verify `nvidia-smi`, Docker GPU access, and self-test behavior before accepting new rentals. + +## Related pages + +| Topic | Read next | +| --- | --- | +| Contract and offer end dates | [Hosting Overview](/host/hosting-overview#offer-end-date) | +| Reliability impact of downtime | [Reliability & Uptime](/host/reliability-uptime) | +| Removing a machine entirely | [Removing or Recreating Machines](/host/removing-recreating-machines) | diff --git a/host/market-metrics.mdx b/host/market-metrics.mdx index 12db6a15..d6de7571 100644 --- a/host/market-metrics.mdx +++ b/host/market-metrics.mdx @@ -2,119 +2,139 @@ title: "GPU Market Metrics" sidebarTitle: "Market Metrics" description: "Track GPU supply, demand, pricing, and locations across the Vast marketplace" +personas: + - business-owner + - pro-operator --- -Vast provides real-time and historical data on GPU supply, demand, pricing, and geographic distribution across the marketplace. You can use this data to make informed decisions about pricing your machines and understanding demand for specific GPU types. +
BusinessPro Operator
-## Accessing the data +Use market metrics to compare GPU supply, demand, pricing, and geography before setting host prices. -There are three ways to access market metrics: +## Access -1. **Dashboards** at [cloud.vast.ai/host/market/](https://cloud.vast.ai/host/market/) -2. **CLI** via `vastai metrics` subcommands (requires `pip install vastai`, v1.0.4+) -3. **REST API** endpoints (see [API reference](#api-endpoints) below) +All Vast market metric methods require a host-enabled account. Some views also require the account to have at least one registered machine. -All three require a **host API key**. Regular client keys will receive a `401` error. +| Method | Where | +| --- | --- | +| Dashboard | [cloud.vast.ai/host/market](https://cloud.vast.ai/host/market/) | +| CLI | `vastai metrics ...` | +| REST API | `/api/v0/metrics/gpu/...` | + +Accounts not enabled for hosting can receive `401` errors. If the dashboard says "Register a machine to view market stats" or "Market stats are available once you have at least one registered machine," register a host machine first, then return to the page. + +If you are evaluating hardware before registering a machine, public third-party dashboards can help with rough comparison, but Vast's dashboard, CLI, and API are the source of truth for host market metrics. + +## Dashboard Views -## Dashboards +- **Availability & Pricing**: historical supply/demand and P10/median/P90 prices. +- **GPU Overview**: current supply/demand, 30-day rental rate, and median price by GPU type. +- **GPU Locations**: geographic distribution. +- If the page is blocked by the registered-machine requirement, use the Vast hosting calculator or public market dashboards for preliminary planning until the host account has a registered machine. -Three views are available at [cloud.vast.ai/host/market/](https://cloud.vast.ai/host/market/): +## GPU Overview Income Estimate -- **Availability & Pricing** — Historical supply/demand counts and P10/median/P90 pricing over time -- **GPU Overview** — Current supply/demand counts, pricing, and stats for each GPU type -- **GPU Locations** — Geographic distribution of GPUs across the platform +In [Host Market](https://cloud.vast.ai/host/market/), open **GPU Overview** to compare demand and pricing by GPU model. -## CLI commands +For a quick compute-only estimate, combine: -Install or upgrade the CLI: +| Column | Use | +| --- | --- | +| `% Rented (30D Avg)` | Approximate utilization for that GPU type over the last 30 days. | +| `$/HR MED` | Median hourly GPU price for that GPU type. | + +```text +average compute revenue per GPU-hour = + $/HR MED x (% Rented (30D Avg) / 100) + +monthly compute estimate = + GPUs x $/HR MED x 720 x (% Rented (30D Avg) / 100) +``` + +Example: if `RTX 5090` shows `% Rented (30D Avg)` of `65.0%` and `$/HR MED` of `$0.361/hr`: + +```text +0.361 x 0.65 = $0.23465 average gross compute revenue per hour per GPU +0.361 x 720 x 0.65 = $168.95 gross compute revenue per month per GPU +``` + +This is a better starting point than price alone because utilization matters. A GPU with a higher hourly price can still earn less if it is rented less often. Treat this as a rough gross compute estimate; it does not include storage, bandwidth, volume revenue, power, cooling, hardware cost, taxes, payout timing, or downtime. + +## CLI + +Install or upgrade: ```bash pip install --upgrade vastai ``` -### Current snapshot — `vastai metrics gpu` - -Returns current supply, demand, and pricing for all GPU types. +Current snapshot: ```bash -vastai metrics gpu # all GPU types +vastai metrics gpu vastai metrics gpu --verified true --datacenter true -vastai metrics gpu --raw # JSON output +vastai metrics gpu --raw ``` -See [full reference](/host/cli/metrics-gpu). +Historical trends: -### Historical trends — `vastai metrics gpu-trends` +```bash +vastai metrics gpu-trends +vastai metrics gpu-trends "RTX 4090" +vastai metrics gpu-trends "RTX 4090, H100_SXM" +vastai metrics gpu-trends all +vastai metrics gpu-trends "RTX 4090" --full +vastai metrics gpu-trends "RTX 4090" --raw +vastai metrics gpu-trends "RTX 4090" --start 1773298800 --end 1773817200 --step 3600 +``` -Returns time-series data for supply, demand, and pricing. Defaults to RTX 5090, 4090, and 3090 over the last 24 hours. +Locations: ```bash -vastai metrics gpu-trends # defaults -vastai metrics gpu-trends "RTX 4090" # single GPU -vastai metrics gpu-trends "RTX 4090, H100_SXM" # multiple GPUs -vastai metrics gpu-trends all # all GPU types -vastai metrics gpu-trends "RTX 4090" --full # all data points (vs sampled ~20) -vastai metrics gpu-trends "RTX 4090" --raw # JSON output -vastai metrics gpu-trends "RTX 4090" --start 1773298800 --end 1773817200 --step 3600 +vastai metrics gpu-locations +vastai metrics gpu-locations --verified true --datacenter true +vastai metrics gpu-locations --gpu "RTX 4090, H100_SXM" +vastai metrics gpu-locations --rented false +vastai metrics gpu-locations --raw ``` -See [full reference](/host/cli/metrics-gpu-trends). +References: [metrics gpu](/host/cli/metrics-gpu), [metrics gpu-trends](/host/cli/metrics-gpu-trends), [metrics gpu-locations](/host/cli/metrics-gpu-locations). -### Location data — `vastai metrics gpu-locations` +## Public Comparison Sources -Returns geographic distribution of GPUs. +For planning before you have dashboard access, compare multiple sources: -```bash -vastai metrics gpu-locations # unfiltered -vastai metrics gpu-locations --verified true --datacenter true # datacenter-verified only -vastai metrics gpu-locations --gpu "RTX 4090, H100_SXM" # specific GPU types -vastai metrics gpu-locations --rented false # only unrented -vastai metrics gpu-locations --raw # JSON output -``` +- [Vast GPU market prices](https://vast.ai/pricing) +- [Vast hosting earnings calculator](https://vast.ai/hosting/calculator) +- [500.farm Vast charts](https://500.farm/vastai/charts/) -See [full reference](/host/cli/metrics-gpu-locations). +Third-party dashboards are useful for trend checks, but they can be delayed, filtered differently, or incomplete compared with Vast's own host market data. -## API endpoints +## API -You can also query the data directly via the REST API. Pass your host API key as a Bearer token. +Pass your host API key as a Bearer token. | Endpoint | Description | | --- | --- | -| `GET /api/v0/metrics/gpu/current/` | Current snapshot of all GPU types | -| `GET /api/v0/metrics/gpu/history/` | Time-series supply/demand and pricing | -| `GET /api/v0/metrics/gpu/locations/` | Geographic distribution | - -### Query parameters +| `GET /api/v0/metrics/gpu/current/` | Current GPU snapshot | +| `GET /api/v0/metrics/gpu/history/` | Historical supply, demand, and pricing | +| `GET /api/v0/metrics/gpu/locations/` | GPU geography | -| Parameter | Applies to | Description | -| --- | --- | --- | -| `verified` | current, history | `yes`, `no`, or `all` | -| `hosting_type` | current, history | `secure_cloud`, `community`, or `all` | -| `gpu_name` | history | GPU name (e.g. `RTX 4090`) | -| `start` | history | Unix timestamp for range start | -| `end` | history | Unix timestamp for range end | -| `step` | history | Seconds between data points (e.g. `3600` for hourly) | +Common parameters: -### Examples +| Parameter | Values | +| --- | --- | +| `verified` | `yes`, `no`, `all` | +| `hosting_type` | `secure_cloud`, `community`, `all` | +| `gpu_name` | GPU name, such as `RTX 4090` | +| `start`, `end` | Unix timestamps | +| `step` | Seconds between history points | ```bash -# Current snapshot — verified datacenter GPUs curl -H "Authorization: Bearer $API_KEY" \ "https://console.vast.ai/api/v0/metrics/gpu/current/?verified=yes&hosting_type=secure_cloud" - -# Historical trends — RTX 4090, hourly steps -curl -H "Authorization: Bearer $API_KEY" \ - "https://console.vast.ai/api/v0/metrics/gpu/history/?gpu_name=RTX+4090&verified=yes&start=1773298800&end=1773817200&step=3600" - -# Locations -curl -H "Authorization: Bearer $API_KEY" \ - "https://console.vast.ai/api/v0/metrics/gpu/locations/" ``` -## Data freshness - -Market metrics data is updated every 5 minutes. Location data is cached for up to 2 hours since it changes less frequently. - -### Rate limits +## Freshness And Limits -Each endpoint allows up to **5 requests per second** per user. For large historical queries, the backend automatically adjusts the resolution to keep response sizes reasonable. +Market metrics update every 5 minutes. Location data can be cached for up to 2 hours. Each endpoint allows up to 5 requests per second per user. diff --git a/host/network-ports.mdx b/host/network-ports.mdx new file mode 100644 index 00000000..74b7fd72 --- /dev/null +++ b/host/network-ports.mdx @@ -0,0 +1,123 @@ +--- +title: "Network & Ports" +sidebarTitle: "Network & Ports" +description: "Port forwarding, CGNAT, public IP, IPv6, and host connectivity guidance." +"canonical": "/host/network-ports" +personas: + - pro-operator + - headless-operator +--- + +
Pro OperatorHeadless / DC
+ +Configure public inbound networking before verification. Vast hosts need direct ports reachable from outside the host LAN. + + +## Ports Per GPU + +Self-test requires at least 3 direct ports per listed GPU. For production hosting, plan for about 100 direct ports per listed GPU. One instance can map up to 64 ports, and a larger range gives headroom while old mappings are being released and new instances are starting. + +| Listed GPUs | Minimum | Practical example | +| --- | ---: | --- | +| 1 | 3 | `40000-40099` | +| 4 | 12 | `40000-40399` | +| 8 | 24 | `40000-40799` | + +The forwarded router/firewall range must match the range configured in the host software. The 100-port guidance is practical headroom, not a self-test minimum. + + +## TCP And UDP + +Forward both TCP and UDP for the configured direct port range. TCP-only forwarding can pass some checks while still failing renter workloads that need UDP. + +## Set The Host Port Range + +In the Host Installer Wizard, enter the same range you forwarded. + +For the raw installer: + +```bash +sudo python3 ./install "$VAST_HOST_KEY" --ports 40000 40799 +``` + +For an installed host: + +```bash +echo -n "40000-40799" | sudo tee /var/lib/vastai_kaalia/host_port_range +sudo systemctl restart vastai.service +sudo tail -n 50 /var/lib/vastai_kaalia/kaalia.log | grep host_port_range +``` + +The log should show `host_port_range: 40000 40799`. + + +## CGNAT, Starlink, IPv6-Only, Or No Public IP + +Vast instances need public inbound TCP and UDP reachability. CGNAT, double NAT without a real public forwarding path, many residential Starlink setups, incompatible IPv6-only service, blocked inbound ports, or no public inbound IP are not supported for hosting because renters and self-test must be able to reach the machine directly. + +Compare your router WAN IP with your public IP and test from outside the LAN. If the router does not have a public inbound address or cannot forward the configured ports to the host, use a network connection that provides public inbound connectivity before listing the machine. + + +## Test Ports From Outside The LAN + +Same-LAN tests can be misleading because of NAT hairpinning. Test from a different network, such as mobile data, a cloud VM, or another external machine. + +On the host, start a temporary TCP listener and confirm it is listening: + +```bash +sudo python3 -m http.server 40000 --bind 0.0.0.0 +sudo ss -ltnp | grep ':40000' +``` + +From macOS or Linux outside the LAN: + +```bash +curl -v http://PUBLIC_IP:40000/ +nc -vz PUBLIC_IP 40000 +``` + +From Windows PowerShell outside the LAN: + +```powershell +Test-NetConnection PUBLIC_IP -Port 40000 +curl.exe http://PUBLIC_IP:40000/ +``` + +You can also use a public TCP checker such as [portchecker.co](https://portchecker.co/) for a quick outside-LAN TCP check. + +For UDP, watch for packets on the host: + +```bash +sudo tcpdump -ni any udp port 40000 +``` + +Then send a UDP packet from macOS or Linux outside the LAN: + +```bash +printf 'vast-port-test\n' | nc -u -w2 PUBLIC_IP 40000 +``` + +From Windows PowerShell outside the LAN: + +```powershell +$udp = New-Object System.Net.Sockets.UdpClient +$bytes = [Text.Encoding]::ASCII.GetBytes("vast-port-test") +$udp.Send($bytes, $bytes.Length, "PUBLIC_IP", 40000) +$udp.Close() +``` + +Stop temporary listeners before running the installer. + +During self-test, troubleshoot the exact public IP and port reported by the CLI, such as `https://PUBLIC_IP:PORT/progress`. + +## Connectivity Failures + +Self-test timeouts and `Port Networking Issues` usually mean a network-path problem. [`vericode=8`](/host/glossary#vericode) means a host-facing machine error is present; use the exact visible message to confirm whether it is the port-networking case. See [Self-Test Reference: vericode=8](/host/self-test-reference#vericode-8) and [no response for 120s](/host/self-test-reference#no-response-120s). + +## Related Pages + +| Topic | Read next | +| --- | --- | +| Hardware/network fit | [Supported Hardware](/host/supported-hardware) | +| Self-test | [How to Self-Test](/host/how-to-self-test) | +| Failure reference | [Self-Test Reference](/host/self-test-reference) | diff --git a/host/not-in-search.mdx b/host/not-in-search.mdx new file mode 100644 index 00000000..907badf8 --- /dev/null +++ b/host/not-in-search.mdx @@ -0,0 +1,69 @@ +--- +title: "Why Isn't My Machine in Search?" +sidebarTitle: "Not in Search" +description: "Troubleshooting machine visibility in marketplace search." +"canonical": "/host/not-in-search" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist +--- + +
All host personas
+ +Use this page when a machine is listed but does not appear where you expect in marketplace search. + +## Listed and visible are different things + +Separate "listed" from "visible in a normal broad search." Marketplace search is not a complete inventory dump. It limits result sets, ranks and groups similar offers, tries to avoid race conditions, and usually shows the best matches first. Search can be affected by ranking, grouping, filters, verification state, offer state, rental state, reliability, price, supply and demand, or machine errors. + + +## Check the machine directly + +Start in the Machines page for the host account. Confirm that the machine is online, listed, has active offers, has no red machine errors, and is not currently rented in a way that hides the offer you expect to see. Also confirm required details such as direct ports, RAM, upload speed, and download speed are populated, and that reliability is above the relevant gate. + +Confirm the CLI is authenticated to the same host-enabled account: + +```bash +vastai show user +``` + +Then search directly by machine ID: + +```bash +vastai search offers 'machine_id=' --limit 200 +``` + +If you need to ignore the CLI's default search filters while troubleshooting, add `-n`: + +```bash +vastai search offers -n 'machine_id=' --limit 200 +``` + +You can also keep the normal command form but explicitly include the offer states that commonly hide a machine: + +```bash +vastai search offers 'machine_id= rentable=any rented=any verified=any external=any' --limit 200 +``` + +If the machine looks healthy there but is hard to find in marketplace search, narrow the search by GPU model, region, verification state, price range, and rentable/rented state. The machine can be listed correctly but still not appear in a broad search because other machines rank higher, similar offers are grouped, or your filters exclude it. + +## Comparing your ranking + +To understand which machines rank above yours, use narrow filters that match your hardware, for example: + +```bash +vastai search offers 'gpu_name=RTX_4090 cpu_ram>257 cpu_ram<258' +``` + +The default Auto Sort combines ranking factors with an element of randomness, so position varies between searches. + +## Related pages + +| Topic | Read next | +| --- | --- | +| Common question index | [Common Host Questions](/host/common-host-questions) | +| Verification state | [Verification Stages](/host/verification-stages) | +| Pricing competitiveness | [Pricing Your Listing](/host/pricing-your-listing) | +| Self-test rentability errors | [Self-Test Failures](/host/self-test-reference#machine-not-rentable) | diff --git a/host/optimization-guide.mdx b/host/optimization-guide.mdx index 3df96417..88a6f39a 100644 --- a/host/optimization-guide.mdx +++ b/host/optimization-guide.mdx @@ -1,156 +1,94 @@ --- title: "Optimize Your Earnings" -description: "A guide to the key settings, tradeoffs, and strategies that determine how much your machines earn. Your hardware is only half the equation; how you configure your listings is the other." +description: "Key listing settings that affect occupancy and revenue." "canonical": "/host/optimization-guide" +personas: + - business-owner + - pro-operator --- - -**Who this is for:** Hosts who have already completed the technical setup process and have machines ready to list. This guide covers the *economic* decisions; that is, the settings and strategies that affect your ranking, occupancy, and revenue. - +
BusinessPro Operator
-## Understanding Your Market Position +Use this page after your machine is ready to list. Hardware matters, but listing settings decide who can find and rent it. -Before tuning any individual setting, it helps to understand where your supply sits relative to demand. Your listing competes with others for customer attention, and your goal is to find the combination of price, availability, and configuration that maximizes your total revenue over time. +## Core Rule -Revenue isn't just about price. It's the product of your **price per hour** and your **occupancy rate** (the percentage of time your machines are rented). A high price with low occupancy can easily earn less than a moderate price with strong utilization. Every setting in this guide affects that balance. +Revenue is price multiplied by utilization. A high price with low occupancy can earn less than a moderate price with steady rentals. -Customers filter and sort listings by a variety of criteria, most commonly by **verification status**, **duration**, and **disk space**. If your listing doesn't pass these filters, it won't appear in search results regardless of how competitive your price is. - - -Think of your listing as a product on a shelf. Price is only one factor; other factors like availability, duration terms, and configuration determine whether customers even see it. - +Before changing settings, compare similar machines by GPU model, GPU count, verification state, region, reliability, storage, bandwidth, and listing duration. ## Duration -Duration controls how long a rental contract can last. It's one of the most impactful settings because it affects both the type of customer you attract and the stability of your revenue. - -**The core tradeoff:** Longer durations tend to increase expected earnings because customers value the guaranteed availability. However, you are locked into your pricing terms for the full contract length. For example, if market rates rise, you can't adjust mid-contract. - -| Short duration | → | Long duration | -| :-- | :-: | --: | -| *Flexibility, lower commitment* | | *Higher expected value, price lock-in* | - -### Customer Segments by Duration - -Different customer types have very different duration preferences. Understanding this helps you target the right segment for your hardware: - -| Customer Type | Typical Duration Need | Notes | -| :-- | :-- | :-- | -| Serverless / automated inference | Short (hours to days) | These workloads are transient. Use short-term instances for this segment. | -| Experimentation & fine-tuning | Medium (days to weeks) | Researchers and developers testing models before committing to long runs. | -| Training runs | Long (weeks to months+) | Customers often prefer very long durations or no maximum. Stable, high-value contracts. | - - -Consider enabling the **auto-extend** feature. This allows rentals to automatically continue beyond their initial term. It reduces churn without requiring you to commit to a longer maximum duration. - - - -**Data center hosts:** Do not set a duration longer than you can reliably honor. Breaking a contract early carries a serious penalty. Only commit to what you can guarantee. - - -## Pricing - -Pricing is the most direct lever for your earnings, but the optimal price isn't simply "as high as possible." Rather, it's the rate that maximizes total revenue given your hardware, competition, and current market conditions. - -**The core tradeoff:** Higher prices increase revenue per rental hour, but reduce occupancy. Lower prices fill your machines faster, but each hour earns less. The optimum sits somewhere in the middle and shifts with supply and demand. - -| Lower price | → | Higher price | -| :-- | :-: | --: | -| *Higher occupancy* | | *Higher per-hour revenue* | - -### On-Demand Pricing - -Your primary listing price. Use market data and your competitors' pricing as a reference point. Consider your hardware's relative performance, your verification status, and your reliability track record, as these all affect how customers evaluate your listing against alternatives. +Longer durations can attract renters who value guaranteed availability, but they also lock you into the accepted price and end date. -### Interruptible Pricing (Min Bid Price) +| Shorter duration | Longer duration | +| --- | --- | +| More flexibility | Better fit for long jobs | +| Easier to reprice | Longer price lock-in | +| Less commitment | Stronger availability promise | -This is one of the most commonly misconfigured settings. The Min Bid Price is a **floor** for the bidding system, *not* an on-demand price. Customers bid above this floor, and competition among bidders will drive the actual price higher. +Only offer durations you can honor. Breaking an active rental contract harms renters and can hurt reliability. -Many hosts set their interruptible floor too high because they treat it like their on-demand rate. This is a mistake. When your machines are idle, you earn nothing. A well-set interruptible floor ensures your GPUs are working even during low-demand periods. +## Price -**Recommended strategy:** Set your Min Bid Price close to your **cost of power when the GPU is under load**. At this floor, you're at least covering your electricity costs during idle periods. Bidders will typically pay above this floor, so your actual interruptible revenue will be higher. If no one bids, consider running your own background jobs on the otherwise-idle hardware. +Use comparable listings and [market metrics](/host/market-metrics). Adjust from actual utilization, not from the highest visible price. - -Your floor price should **not** be close to your expected actual rental price. The bidding mechanism will bring the price up, so you set the minimum you'd accept, not the price you hope to get. - +| If this happens | Consider | +| --- | --- | +| Machine is healthy but idle | Lowering price or improving listing terms | +| Machine is always rented | Testing a modest price increase | +| Similar machines are cheaper | Matching the market until reliability/history improves | +| Demand spikes | Rechecking price after existing contracts end | -### Disk Pricing +Price changes affect future rentals only. Existing rental contracts keep the terms accepted at rental time. -Disk pricing depends on how much storage you have relative to what clients need: +## Interruptible Minimum Bid -| Situation | Approach | -| :-- | :-- | -| You have **excess disk** capacity | Set disk pricing closer to your amortized cost. Storage is not your bottleneck. | -| You are **under-provisioned** on disk | Set disk pricing higher than you might expect. This prevents stopped instances from consuming all your storage and starving active rentals. | +The minimum bid is a floor, not your expected on-demand price. Set it near the lowest price you are willing to accept, often close to loaded power cost. -### Internet Bandwidth Pricing +If the floor is too high, GPUs may sit idle. If it is too low, interruptible work may not cover operating cost. -This depends entirely on your network setup. If you have fast, unmetered upstream internet, you can price bandwidth low or even at zero, as this is a strong competitive advantage. If your network has QoS controls or internal metering, set a reasonable bandwidth price to prevent any single client from saturating your connection and degrading service for others. +## Disk And Bandwidth -## Min GPU Configuration +| Setting | Guidance | +| --- | --- | +| Disk price | Price lower if storage is abundant; price higher if disk is scarce. | +| Bandwidth price | Price lower for fast, unmetered links; price higher if bandwidth is metered or capacity-limited. | -The Min GPU setting controls the smallest GPU partition you offer. For an 8-GPU machine, this determines whether customers can rent 1, 2, 4, or 8 GPUs at a time. It's a critical lever for balancing demand capture against utilization efficiency. +Do not let stopped instances, cached images, or rented volumes consume the disk needed for active rentals. -**The core tradeoff:** Smaller minimums (e.g., 1x) let you serve more customers and fill individual GPU slots, but create fragmentation. Partial rentals can leave awkward, hard-to-fill gaps, leading to underutilization. Larger minimums (e.g., 8x) eliminate fragmentation but limit your audience to customers who need a full node. +## Min GPU -| Min GPU = 1 | → | Min GPU = 8 | -| :-- | :-: | --: | -| *More renters, more fragmentation* | | *Fewer renters, cleaner utilization* | +`min_gpu` controls the smallest GPU slice renters can choose. -### How It Works +| Smaller `min_gpu` | Larger `min_gpu` | +| --- | --- | +| More renters can use the machine | Better fit for full-node workloads | +| Higher fragmentation risk | Lower fragmentation | +| Can improve occupancy | Can miss small-job demand | -When you set a Min GPU value, the platform creates listings at all powers of two from your minimum up to the total GPUs in the machine. For example, on an 8-GPU machine with Min GPU set to 2, listings are created for 2x, 4x, and 8x configurations. +For fleets, mix `min_gpu` values across machines instead of setting every host the same way. -### Recommended Approach +## Keep The Listing Competitive -If you operate multiple machines, distribute your Min GPU settings across them rather than using the same value everywhere. A healthy mix might look like this: +Basics that affect revenue: -| Machine | Min GPU | Available Listings | -| :-- | :-- | :-- | -| Machine A | 1 | 1x, 2x, 4x, 8x | -| Machine B | 2 | 2x, 4x, 8x | -| Machine C | 4 | 4x, 8x | -| Machine D | 8 | 8x only | +- Keep drivers current before listing. +- Run self-test after setup and major changes. +- Maintain reliability above the verification gate. +- Avoid unplanned reboots, disconnects, and overheating. +- Highlight datacenter status only when it is real and approved. -This approach lets you capture demand across all configuration sizes while keeping fragmentation in check. Market data suggests there is meaningful demand for both 1x and 8x configurations, with approximately twice as much search volume for 8x rentals as for 1x. +## Quick Reference - -Avoid setting *all* machines to 1x, as this leads to severe fragmentation and reduced overall utilization. Conversely, all-8x can work if competitors in your GPU class are highly fragmented and there's strong demand for full nodes, but it's generally not the default recommendation. - - -The optimal mix depends on current market conditions, so be sure to consider what your competitors are offering and where unmet demand exists. Review your utilization patterns and adjust periodically. - -## Keeping Your Machines Competitive - -Settings alone won't help if your machines aren't reliable and up to date. A few operational basics make a significant difference in your listing's performance. - -### Driver Updates - -Ensure you have the latest NVIDIA drivers installed at the time of listing. Outdated drivers can cause compatibility issues with customer workloads and result in failed rentals. - -### Testing - -Test your machines before listing and after any configuration changes. A machine that appears available but fails when rented wastes a customer's time and yours, so you lose the rental and risk a negative reliability signal. - -### Verification - -Verification is one of the most common filters customers apply when searching. Ensuring your machines are verified significantly increases your visibility in search results. - -### Data Center Hosts - -If you operate out of a data center, consider highlighting any certifications you hold (e.g., Tier 4 data center, HIPAA compliance) as differentiators. These can be meaningful selling points for enterprise and compliance-sensitive customers. - ---- +| Setting | Main question | +| --- | --- | +| Duration | How long can you reliably commit? | +| On-demand price | What price balances occupancy and hourly revenue? | +| Min bid | What is your acceptable floor? | +| Disk price | Is storage abundant or scarce? | +| Bandwidth price | Is bandwidth unmetered or constrained? | +| `min_gpu` | Do you want more small renters or cleaner full-node utilization? | +| Auto-extend | Should same-price or lower-price rentals continue automatically? | -## Quick Reference: Settings at a Glance - -| Setting | Key Question | Watch Out For | -| :-- | :-- | :-- | -| **Duration** | How long can you reliably commit? | Over-committing (especially data centers); price lock-in on long contracts | -| **On-Demand Price** | What rate balances occupancy and revenue? | Pricing in a vacuum without checking competitors | -| **Min Bid Price** | What's your absolute floor (cost of power)? | Setting it like an on-demand price; leaving GPUs idle instead | -| **Disk Price** | Are you over- or under-provisioned on storage? | Under-pricing when disk-constrained, causing starvation | -| **Bandwidth Price** | Is your network metered or unmetered? | Zero pricing on a metered connection | -| **Min GPU** | What's the right fragmentation vs. demand balance? | All machines at 1x (fragmentation) or all at 8x (missed demand) | -| **NVIDIA Drivers** | Are drivers current at time of listing? | Outdated drivers causing failed rentals | -| **Auto-Extend** | Is it enabled? | Forgetting to turn it on | +For detailed pricing examples, see [Pricing Your Listing](/host/pricing-your-listing) and [Earnings & Pricing Model](/host/earning). diff --git a/host/payment.mdx b/host/payment.mdx index 3a231854..c7376f6a 100644 --- a/host/payment.mdx +++ b/host/payment.mdx @@ -1,178 +1,118 @@ --- -title: "Host Payouts" +title: Host Payouts +sidebarTitle: "Host Payouts" createdAt: "Mon Jan 13 2025 21:20:40 GMT+0000 (Coordinated Universal Time)" updatedAt: "Mon Jan 13 2025 21:31:15 GMT+0000 (Coordinated Universal Time)" canonical: "/host/payment" +personas: + - business-owner --- -This guide explains how Vast.ai host payouts work, when invoices are generated, when payments are sent, and what to do if you experience a payout issue. +
Business
-## Available Payout Methods +Manage payouts from **Earnings** in the console. For tax forms and withholding, see the [Tax Guide for Hosts](/host/guide-to-taxes). -Vast.ai currently supports payouts through: +## Earnings Page -- Wise -- PayPal -- Stripe - -Hosts are responsible for ensuring they can receive business-to-business (B2B) payments through their selected payout provider in their country or region. - -Before selecting a payout method, verify that: - -- The service is available in your country. -- Your account is fully verified. -- Your account can receive business payments. -- Any required identity, tax, or business verification has been completed. -- There are no restrictions, limitations, or compliance holds on your account. - -Please refer to your payout provider's documentation for regional availability, account requirements, and verification requirements. - -**Note:** Vast.ai is not responsible for delays, restrictions, compliance reviews, account limitations, or rejected payments imposed by Wise, PayPal, Stripe, or any other financial institution. - ---- - -## Payout Schedule - -### Minimum Payout Threshold - -Your account must accumulate at least **\$20 USD** before an invoice can be generated. - -Balances below \$20 USD will automatically roll forward until the minimum threshold is reached. - -### Invoice Generation - -Invoices are generated weekly on **Fridays**. - -To generate an invoice: - -- You must have a valid payout method connected to your Vast.ai account. -- Your account balance must meet the \$20 minimum payout threshold. - -### Payment Timeline - -It typically takes up to **two weeks to receive your first payout**. - -Example: +![](/images/console-earning.webp) -- Week 1: Earnings accrue. -- Friday: Invoice is generated and marked as **Pending**. -- Week 2 Friday: Payment is submitted to your selected payout provider. -- Funds are then processed by Wise, PayPal, or Stripe and may take additional time to appear in your account depending on the provider and your region. +The Earnings page shows: -### Invoice Statuses +- **Current Balance**: earned but not yet paid. +- **Total Earnings**: lifetime host, referral, and template earnings. +- **Total Rental Earnings**: lifetime machine-rental earnings. +- **Total Referral Earnings**: lifetime referral earnings. +- **Payout History**: generated payout records and invoices. -#### Pending +## Payout Methods -A Pending invoice has been generated and is scheduled for payment during the next payout cycle. +Vast currently supports: -Invoices generated on Friday are generally scheduled to be paid on the following Friday. - -#### Paid - -An invoice is marked as Paid once Vast.ai has submitted the payout to your selected payout provider. - -The Paid status reflects the status of the payout within Vast.ai's billing system only. It does not indicate whether the funds have completed processing within Wise, PayPal, or Stripe. - -After a payout has been submitted, your payout provider may require additional processing time before funds appear in your account. - ---- - -# Frequently Asked Questions - -## When will I get paid? - -It typically takes up to two weeks to receive your first payout. - -Invoices are generated weekly on Fridays. Once an invoice is generated, it enters a pending state and is scheduled for payment during the following Friday payout cycle. - -After the payout is submitted, your payout provider may require additional processing time before funds appear in your account. - -## Why does my invoice show "Paid" when I haven't received the funds yet? - -The Paid status indicates that Vast.ai has submitted the payout to your selected payout provider. - -The Paid status only reflects the payout status within Vast.ai's billing system. It does not indicate whether the funds have completed processing within Wise, PayPal, or Stripe. +- Wise +- PayPal +- Stripe -Before contacting support, please verify: +Before selecting a method, confirm that the provider works in your region, your account is verified, and it can receive business payments. -- Your payout account is active and fully verified. -- Your payout provider is not requesting additional documentation. -- There are no account limitations, restrictions, compliance reviews, or holds. -- Your account can receive business payments for services provided. +## Payout Account -If you have confirmed your payout account is in good standing and still have not received your funds after a reasonable processing period, please contact support for further review. +Use **Earnings > Payout Account** to connect, view, remove, or switch payout providers. -## I see my invoice is marked as Pending. What does this mean? +Only one payout account can be active at a time. To switch accounts, first deactivate the current account, then activate the new one. -A Pending invoice has been generated but has not yet entered the payment phase. + +Vast is not responsible for payout delays, account holds, compliance reviews, or rejected payments from Wise, PayPal, Stripe, or other financial institutions. + -Invoices are generated every Friday and are generally scheduled to be paid on the following Friday. +## Payout Schedule -Once Vast.ai submits the payout to your selected payout provider, the invoice status will change to Paid. +| Item | Rule | +| --- | --- | +| Minimum payout | $20 USD | +| Invoice generation | Fridays around 12:00 PM Pacific time | +| Payment timing | Usually 2 to 4 weeks from earnings being posted, depending on region and payout method | +| Paid status | Vast submitted the payout; the provider may still be processing it | -## Can Vast.ai send my payout via direct bank transfer (ACH, SWIFT, wire transfer, etc.)? +Your balance rolls forward until it reaches the $20 minimum and a valid payout account is active. -No. + +## Payout History And Invoices -Vast.ai only supports payouts through: +Start from **Account > Earnings**. -- Wise -- PayPal -- Stripe +Use the **Earnings Chart** controls to download CSV or PDF earnings data when available. -Direct bank transfers, ACH payments, wire transfers, and SWIFT payments are not available. +Use **Payout History** to: -## My account is not generating invoices. +- Filter payout records by date. +- Download CSV or PDF payout records and invoices. +- Review invoice status. -Invoices are only generated when: +Use **Payout Account** to manage connected payout providers. It is not the invoice export area. -- A payout method is connected to your Vast.ai account. -- Your balance has reached the \$20 minimum payout threshold. +For billing-history invoices with itemized usage charges, open **Billing**, use **Generate Billing History**, and check **Include Charges**. -If either requirement is not met, an invoice will not be generated. +To customize invoice details, update **Settings > Invoice Information**. -Please verify that your payout method has been connected correctly in your account settings. +## Frequently Asked Questions -## Can I generate an invoice manually? + +### When will I get paid? -### Customizing Invoice Information +Plan for 2 to 4 weeks from when earnings are posted. Invoices are generated weekly on Fridays around 12:00 PM Pacific time. Payment is generally scheduled about two weeks after posting, then the payout provider may need more time to settle the funds. -Hosts may customize the information that appears on invoices by navigating to: + +### What does Pending mean? +The invoice was generated but has not entered the payment phase. It changes to Paid after Vast submits the payout. -### Downloading Invoices and Payout Records + +### Can Vast send direct bank transfers? -To view and download invoices, navigate to: +No. Vast supports Wise, PayPal, and Stripe. Direct bank transfers, ACH, wire, and SWIFT payouts are not available. -`Earnings → Payout History` + +### Why are invoices not generating? -From the Payout History section, you can: +Invoices require both an active payout account and at least $20 in balance. -- View historical payouts -- Filter records by date range -- Download payout records in CSV format -- Download payout records in PDF format + +### Can I generate an invoice manually? -## How much can I make hosting on Vast.ai? +You can download payout records from **Account > Earnings > Payout History**. For billing history, use **Billing > Generate Billing History** with **Include Charges** enabled. -Host earnings depend on several factors, including: + +### How can I have earnings as a Vast user? -- Hardware specifications -- Pricing -- Uptime and reliability -- Geographic location -- Current customer demand +Hosts earn from machine rentals. Users may also earn referral or template income; see the [referral program](/guides/reference/referral-program). -To understand current market pricing and utilization trends, visit the [Market Stats](https://cloud.vast.ai/host/market/) page. -You must be signed in as a host with at least one registered machine to access this page. Alternatively, check out [GPU market prices](https://vast.ai/pricing) or the [Host Earnings Calculator](https://vast.ai/hosting/calculator). +## Earnings Estimates -Market conditions change over time, so earnings cannot be guaranteed. +Host earnings depend on hardware, price, reliability, location, uptime, and demand. Use the [Earnings & Pricing Model](/host/earning), [Market Stats](https://cloud.vast.ai/host/market/), [GPU market prices](https://vast.ai/pricing), and the [Host Earnings Calculator](https://vast.ai/hosting/calculator). diff --git a/host/persona-decision-guide.mdx b/host/persona-decision-guide.mdx new file mode 100644 index 00000000..47547fe7 --- /dev/null +++ b/host/persona-decision-guide.mdx @@ -0,0 +1,76 @@ +--- +title: "Is Vast for Me?" +sidebarTitle: "Is Vast for Me?" +description: "Decision guide for prospective Vast.ai hosts." +"canonical": "/host/persona-decision-guide" +personas: + - hobbyist + - business-owner +--- + +
HobbyistBusiness
+ +Use this before buying hardware, repurposing a machine, or starting an install. + + +Vast is a GPU marketplace, not a managed setup service. Hosts operate their own hardware, OS, drivers, storage, networking, pricing, uptime, and maintenance. + + +## Good Fit + +Hosting is more likely to work if: + +- You have [supported hardware](/host/supported-hardware). +- The host can run native Ubuntu/Linux. +- GPUs are same-type and have enough VRAM. +- Power, cooling, storage, and inbound networking are stable. +- You can troubleshoot Linux, Docker, GPU drivers, PCIe, and routers. +- You can keep the machine online through active rental contracts. +- You are willing to adjust pricing from market data and utilization. + +## Wait Or Reconsider + +| Situation | Why | +| --- | --- | +| Windows or WSL host | Not the supported host OS path. | +| CGNAT, double NAT without public forwarding, blocked inbound ports, incompatible IPv6-only, or no public inbound path | Renters and self-test need direct public TCP/UDP connectivity. | +| Mixed, old, or low-VRAM GPUs | Can fail checks or attract little demand. | +| Expecting Vast to configure the machine | Local hardware, Linux, drivers, storage, and router work are host responsibilities. | +| Expecting guaranteed verification or earnings | Eligibility does not guarantee search placement, rentals, or revenue. | +| Daily gaming/workstation machine | Rented workloads need a stable dedicated host. | + +## Choose Your Path + +| If you are... | Start with | +| --- | --- | +| New to hosting | [Hosting Quickstart](/host/quickstart) | +| Checking a hardware purchase | [Supported Hardware](/host/supported-hardware) and [Earnings Model](/host/earning) | +| Linux-comfortable operator | [Hardware Prep](/host/hardware-prep), then [Install Host Software](/host/installing-host-software) | +| Headless/datacenter operator | [Headless Install](/host/headless-install) and [Fleet Operations](/host/fleet-operations) | +| Evaluating business fit | [GPU Market Metrics](/host/market-metrics), [Pricing Your Listing](/host/pricing-your-listing), and [Host Payouts](/host/payment) | + +## Decision Checklist + +Before continuing, confirm: + +- Hardware, CPU, RAM, PCIe, storage, and network match [Supported Hardware](/host/supported-hardware). +- The host can run native Ubuntu/Linux. +- You will use a dedicated host account. +- You can forward enough direct TCP and UDP ports. +- You can keep the machine powered, cooled, stable, and online during rentals. +- You understand earnings depend on demand, price, reliability, and utilization. +- You know which issues are host responsibilities and which belong with Vast support. + +## Common Questions + +| Question | Read | +| --- | --- | +| What hardware is supported? | [Supported Hardware](/host/supported-hardware) | +| What should I avoid? | [Unsupported setups](/host/supported-hardware#unsupported-or-discouraged-setups) | +| Do I need a separate host account? | [Account & Hosting Agreement](/host/account-hosting-agreement#separate-host-account) | +| What if I do not have a public IP? | [Network & Ports](/host/network-ports#cgnat-starlink-ipv6) | +| How should I price listings? | [Pricing Your Listing](/host/pricing-your-listing) | +| How do hosts get paid? | [Host Payouts](/host/payment) | +| What goes to support? | [Errors & Diagnostics](/host/common-errors-diagnostics#escalate-support) | + +For peer help, see [Discord & Community](/host/community). For setup, continue with [Hosting Quickstart](/host/quickstart). diff --git a/host/pricing-your-listing.mdx b/host/pricing-your-listing.mdx new file mode 100644 index 00000000..866b02c9 --- /dev/null +++ b/host/pricing-your-listing.mdx @@ -0,0 +1,104 @@ +--- +title: "Pricing Your Listing" +sidebarTitle: "Pricing Your Listing" +description: "Listing optimization, DPH, slicing, bids, and pricing examples." +"canonical": "/host/pricing-your-listing" +personas: + - business-owner + - pro-operator +--- + +
BusinessPro Operator
+ +Use this page when you are ready to list a machine or adjust an offer. Price for total revenue, not just the highest hourly rate. + +## Before You Price + +Confirm: + +- No unresolved hardware, driver, Docker, storage, or network errors. +- Offer end date matches the commitment you can honor. +- Similar machines have been checked in [GPU Market Metrics](/host/market-metrics). +- Gross revenue has been estimated with the [Earnings Model](/host/earning). +- You understand that price changes affect future rental contracts only. + + +## Listing Controls + +| Control | CLI option | Meaning | +| --- | --- | --- | +| On-demand GPU price | `--price_gpu` | Dollars per GPU-hour. | +| Interruptible min bid | `--price_min_bid` | Lowest bid you will accept for interruptible rentals. | +| Reserved discount | `--discount_rate` | Maximum long-term prepay discount. | +| Minimum GPU chunk | `--min_chunk` | Smallest GPU group renters can select. | +| Offer end date | `--end_date` or `--duration` | How long new renters can accept the offer. | +| Disk price | `--price_disk` | Dollars per GB per month for inactive storage. | +| Bandwidth price | `--price_inetu`, `--price_inetd` | Dollars per GB uploaded/downloaded. | +| Volume offer | `--vol_size`, `--vol_price` | Optional separately listed storage. | + +DPH means dollars per hour, usually GPU compute price. + +## Starting Workflow + +1. Compare same GPU model, GPU count, verification state, region, and machine quality. +2. Check P10, median, P90, and `% Rented (30D Avg)` for the last 30 days. +3. Pick an on-demand price for your goal: occupancy, hourly rate, or balance. +4. Set `min_gpu`/`--min_chunk` to `1` unless you have a reason to require larger rentals. +5. Set an interruptible min bid only as a floor, not as your target price. +6. Use reserved discounts only if you can support longer rentals. +7. Set an offer end date you can honor. + +Use [GPU Market Metrics](/host/market-metrics#gpu-overview-income-estimate) to estimate income from both utilization and median price. Price alone is not enough; a machine that rents at a slightly lower rate for more hours may earn more than a high-priced idle listing. + +## Example + +For a 4-GPU machine with comparable prices of P10 `$0.35`, median `$0.45`, and P90 `$0.65` per GPU-hour: + +| Setting | Example | +| --- | --- | +| On-demand price | `$0.45/GPU/hr` | +| Interruptible min bid | `$0.25/GPU/hr` | +| Reserved discount | `0.30` | +| Minimum GPU chunk | `1` | +| Offer end date | Maintenance-safe date | + +```bash +--price_gpu 0.45 \ + --price_min_bid 0.25 \ + --discount_rate 0.30 \ + --min_chunk 1 \ + --end_date 12/31/2026 +``` + +Adjust for your market, operating cost, and risk tolerance. Use the current [vastai list machine](/host/cli/list-machine) syntax. + +## Changing Price Later + +When a renter accepts an offer, its terms become that rental contract. Later changes affect only future rentals. + +| Change | Effect | +| --- | --- | +| Raise or lower price | New renters see the new price; active renters keep the accepted price. | +| Shorten offer end date | New contracts use the new date; active rental end dates do not shorten. | +| Unlist machine | Stops new rentals; active contracts continue. | + +Keep the machine online until the latest active rental end date. + +## When To Adjust + +| Signal | Possible action | +| --- | --- | +| Healthy but idle | Lower price or improve listing fit. | +| Always rented | Test a modest increase for future contracts. | +| Similar machines are cheaper | Reprice or improve machine quality. | +| Poor rentals despite fair price | Check verification, reliability, search visibility, network, storage, and errors. | +| Strong long-running demand | Consider reserved discount and longer end date. | + +## Related + +| Question | Read next | +| --- | --- | +| Revenue model | [Earnings & Pricing Model](/host/earning) | +| Market data | [GPU Market Metrics](/host/market-metrics) | +| CLI listing | [vastai list machine](/host/cli/list-machine) | +| Interruptible bid floor | [vastai set min-bid](/host/cli/set-min-bid) | diff --git a/host/quickstart.mdx b/host/quickstart.mdx new file mode 100644 index 00000000..c449717a --- /dev/null +++ b/host/quickstart.mdx @@ -0,0 +1,22 @@ +--- +title: "Hosting Quickstart" +sidebarTitle: "Hosting Quickstart" +description: "Numbered journey for first-time hosts." +"canonical": "/host/quickstart" +personas: + - hobbyist +--- + +
Hobbyist
+ +Follow this ordered path for a first-time Vast host setup. Each step links to the page with the full detail. + +## Setup Path + +1. **Decide whether Vast fits your hardware and expectations.** Read [Is Vast for Me?](/host/persona-decision-guide) and check your hardware against [Supported Hardware](/host/supported-hardware). +2. **Create a dedicated host account and accept the hosting agreement.** Start from the official [Vast host setup page](https://cloud.vast.ai/host/setup/), then see [Account & Hosting Agreement](/host/account-hosting-agreement) if the account flow gets stuck. +3. **Prepare the machine.** Set up Ubuntu, drivers, and BIOS with [Hardware Prep](/host/hardware-prep), disks with [Storage Setup](/host/storage-setup), and port forwarding with [Network & Ports](/host/network-ports). +4. **Install the Vast host software with the Host Installer Wizard (TUI).** Copy the current command from the official [Vast host setup page](https://cloud.vast.ai/host/setup/), then use [Installing Host Software](/host/installing-host-software#host-installer-wizard) to understand the install flow. Running over SSH only? Use the [Headless Install Path](/host/headless-install). +5. **List the machine.** Set your price and terms with [Pricing Your Listing](/host/pricing-your-listing). +6. **Run the self-test.** Follow [How to Self-Test](/host/how-to-self-test), and decode any failures with the [Self-Test Reference](/host/self-test-reference). +7. **Understand verification and first-rental expectations.** Read [Understanding Verification](/host/understanding-verification) and [First 24 Hours After Install](/host/first-24-hours). diff --git a/host/reliability-uptime.mdx b/host/reliability-uptime.mdx new file mode 100644 index 00000000..cd885b7b --- /dev/null +++ b/host/reliability-uptime.mdx @@ -0,0 +1,54 @@ +--- +title: "Reliability & Uptime" +sidebarTitle: "Reliability & Uptime" +description: "Reliability score, uptime expectations, and common score drops." +"canonical": "/host/reliability-uptime" +personas: + - pro-operator + - headless-operator +--- + +
Pro OperatorHeadless / DC
+ +Understand reliability score, uptime expectations, common score drops, and practical recovery steps. + + +## Why does reliability drop after rentals, reboots, or disconnects? + +Reliability can drop when the platform observes instability, failed starts, disconnections, downtime, network failures, driver or GPU failures, thermal or power events, or active rental disruption. Short rentals can expose startup and runtime failures quickly. Reboots or disconnects during active commitments can hurt reliability. + +If you must take the machine offline, minimize the offline time. The score also takes the machine's average earnings into account: machines with lower earnings are penalized less for offline time. + + +## How quickly does reliability recover? + +Reliability recovery depends on sustained stable operation and the machine's recent incident history. Some error messages can clear after the underlying issue is resolved, but reliability score recovery should not be treated as an instant reset. Keep the machine stable and error-free, then monitor the score over time. + + +## What should I do when reliability is at or below 0.9? + +Reliability greater than 0.9 is a self-test and verification gate. If the machine is at or below the threshold, the normal self-test can fail before renting the temporary instance and the machine will not enter the verification pool. Keep the machine stable, resolve recent hardware, network, driver, daemon, thermal, power, or rental-disruption issues, then retry once the score recovers. + +The gate exists so machines that have not been online for a sufficient amount of time are not put through verification testing. Above the gate, higher reliability can make verification more likely because the automated process has more confidence in the machine. Sustained instability after verification can cause a verified machine to be deverified. + +Do not use `--ignore-requirements` as a verification shortcut. It is only useful for advanced runtime validation, and the machine still needs enough direct ports for the runtime test to work. + + +## Will restarting lower reliability or remove verification? + +Reliability does not directly determine verification state by itself. Verification is separate from the reliability score, but outages, failed starts, driver issues, network drops, or other instability can reduce reliability and may eventually contribute to deverification if the machine becomes unhealthy. Plan restarts around active contracts and use maintenance windows whenever possible. + + +## Preventing score drops + +- Do not take the machine offline during active rental contracts; plan work through [Maintenance Windows](/host/maintenance-windows). +- Disable unattended kernel and NVIDIA driver updates; see [automatic updates](/host/maintenance-windows#automatic-updates). +- Keep thermals, power delivery, and network stability healthy under sustained load. + +## Related pages + +| Topic | Read next | +| --- | --- | +| How reliability affects verification | [Understanding Verification](/host/understanding-verification) | +| Self-test preflight gates | [How to Self-Test](/host/how-to-self-test) | +| Machine errors | [Common Errors & Diagnostics](/host/common-errors-diagnostics) | diff --git a/host/removing-recreating-machines.mdx b/host/removing-recreating-machines.mdx new file mode 100644 index 00000000..40d77358 --- /dev/null +++ b/host/removing-recreating-machines.mdx @@ -0,0 +1,45 @@ +--- +title: "Removing or Recreating Machines" +sidebarTitle: "Remove or Recreate" +description: "Unlisting, deleting, and recreating host machines." +"canonical": "/host/removing-recreating-machines" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist +--- + +
All host personas
+ +Use this page when you need to unlist, delete, recreate, or replace a host machine. + + +## How do I unlist, delete, or recreate a machine? + +Unlisting stops new rental contracts from being created, but it does not end existing contracts. Deleting or recreating a machine should only happen after active commitments and stored workloads are understood. Recreating may be useful after a failed install, stale record, or major hardware change, but do not use undocumented reset or recovery flags unless Vast support specifically instructs you to do so. + +From the CLI, see [vastai unlist machine](/host/cli/unlist-machine), [vastai delete machine](/host/cli/delete-machine), and [vastai cleanup machine](/host/cli/cleanup-machine). + + +## What changes when I swap GPUs or other hardware? + +Hardware changes can affect verification, search visibility, offers, and client expectations. Reducing GPU count, RAM, disk, or advertised capacity can trigger deverification or contract problems. Adding or replacing GPUs may require the host daemon or backend to refresh machine specs, and you may need to rerun self-test. The same-GPU-type guideline still applies; see [Supported Hardware](/host/supported-hardware). + + +## Uninstalling the host software + +To uninstall, use the Vast uninstall script located at [https://s3.amazonaws.com/vast.ai/uninstall](https://s3.amazonaws.com/vast.ai/uninstall). Make sure active rental contracts have ended and the machine is unlisted first. + + +## What should I do with a ghost machine? + +First inspect whether the machine has active contracts, stored instances or volumes, visible offers, or backend state preventing deletion. Use the normal unlist/delete/cleanup path only when safe. If a stale backend record remains after normal cleanup, collect the account context, screenshots or CLI output, and relevant logs, then contact Vast support. + +## Related pages + +| Topic | Read next | +| --- | --- | +| Honoring active contracts first | [Maintenance Windows](/host/maintenance-windows) | +| Verification impact of changes | [Verification Stages](/host/verification-stages) | +| Diagnostics for stale records | [Common Errors & Diagnostics](/host/common-errors-diagnostics#escalate-support) | diff --git a/host/self-test-reference.mdx b/host/self-test-reference.mdx new file mode 100644 index 00000000..f1cdd037 --- /dev/null +++ b/host/self-test-reference.mdx @@ -0,0 +1,106 @@ +--- +title: "Self-Test Reference" +sidebarTitle: "Self-Test Reference" +description: "Lookup reference for self-test failures, meanings, and next actions." +"canonical": "/host/self-test-reference" +personas: + - pro-operator + - headless-operator +--- + +
Pro OperatorHeadless / DC
+ +Use this reference to look up self-test failures, understand what they usually mean, and choose the next diagnostic step. Each failure has its own anchor so CLI errors and support replies can link directly to it. + +## Quick Error Lookup + +Use the exact message printed by the CLI. Fix the host-side cause, then rerun self-test. + +| Message or stage | Usually means | Host action | +| --- | --- | --- | +| `Machine not found or not rentable` | The machine is not testable from the current account or has no rentable offer. | Check listing state, active offers, account/API key, reliability, red errors, and missing machine details. | +| `Reliability <= 0.90` | The machine has not met the reliability gate. | Keep the host online and stable; use diagnostic mode only for troubleshooting, not verification proof. | +| `Authorization Error`, `HTTP 401`, or missing permission | The API key cannot inspect the machine or offer. | Use a host-enabled account/API key with the needed machine and offer permissions. | +| `CUDA compatibility` failure | Driver/runtime support is too old for the self-test image. | Update or repair the NVIDIA driver stack, then confirm Docker GPU access. | +| Direct ports below requirement | The machine does not expose enough direct ports for the listed GPUs. | Configure and forward at least 3 TCP/UDP ports per listed GPU for self-test; use more headroom for production hosting. | +| PCIe bandwidth failure or `bad bandwidthtest2` | GPU transfer or PCIe health is below the gate. | Check PCIe lanes, risers, slots, power, thermals, Xid/AER logs, and GPU seating. | +| Upload/download bandwidth failure | Network throughput is below the machine requirement. | Check ISP/datacenter bandwidth, routing, host load, and repeated speed results. | +| VRAM, RAM, or physical CPU core failure | Hardware resources do not meet the listed GPU requirement. | Confirm visible GPUs, host RAM, and at least one physical CPU core per visible GPU. | +| `progress_endpoint_unreachable`, `progress_empty_timeout`, `progress_endpoint_lost`, or `No response for 120s` | The temporary test instance did not provide progress. | Check public TCP reachability, container startup, Docker/NVIDIA runtime, daemon logs, and the support bundle. | +| UDP echo failure | TCP may work, but the public UDP path failed. | Forward UDP for the same direct port range and test from outside the LAN. | +| System requirements failure | The container could not confirm core host resources. | Check RAM, CPU cores, GPU visibility, and container logs in the bundle. | +| ResNet, ECC, or `gpu-burn` failure | GPU compute, memory, ECC, thermal, or stability issue. | Check `nvidia-smi`, Xid/NVRM logs, thermals, power, and rerun after fixing hardware/driver issues. | +| `failed to inject CDI devices` or `nvidia-container-cli` | Docker could not inject GPUs into the container. | Confirm `nvidia-smi` and a normal Docker GPU container both work. | +| `nccl_failed` | Multi-GPU communication failed. | Check CUDA/NCCL compatibility, PCIe/NVLink topology, P2P, Xid logs, and Fabric Manager on NVSwitch systems. | + + +## vericode=8 And Port Networking Issues + +[Glossary: vericode](/host/glossary#vericode) + +`vericode=8` means the machine has a host-facing `error_msg` in the platform state. If the visible message is `Port Networking Issues`, `Port issue`, or similar wording, treat it as a direct-port or connectivity problem. Other `vericode=8` cases can point to Docker/NVIDIA runtime, CDI, PCIe, daemon, or other machine errors, so use the exact console message when choosing the next fix. + +For port-networking cases, common causes include closed external ports, forwarding to the wrong LAN IP, CGNAT or double NAT without a real public forwarding path, no inbound public IP, firewall rules, TCP/UDP mismatch, asymmetric protocol handling, or a stale port range. Test the public IP:port if available, verify router and host firewall rules, then rerun the self-test after fixing the network path. See [Network & Ports](/host/network-ports) for the full connectivity checklist and [Machine Error Reference](/host/machine-errors) for non-port `vericode=8` errors. + + +## No response for 120s + +A `no response` or progress timeout means the CLI could not get usable progress from the temporary self-test instance after it was created. Newer CLI output may show a more specific code such as `progress_endpoint_unreachable`, `progress_empty_timeout`, or `progress_endpoint_lost`. Start with connectivity and startup checks: + +- Inspect the tested external IP:port if the CLI printed one. +- Confirm router or firewall forwarding to the host LAN IP. +- Retry from an outside network to rule out NAT hairpinning; see [testing from outside your LAN](/host/network-ports#test-ports-outside-lan). +- Inspect the diagnostic bundle for instance status, container logs, and daemon logs. + +Other likely causes include container startup failure, Docker/NVIDIA runtime stalls, host daemon stalls, GPU or system hangs, or upload instability. + + +## Machine not found or not rentable + +This can mean the machine is already rented in a way that hides the expected offer, has zero active on-demand offers, is offline or unlisted, is not visible to the account running the test, is deverified or below a threshold, or has offer-side error metadata. + +Start with: + +1. Open the Machines page for the host account. +2. Confirm the machine is listed, online, not currently rented, and has active offers. +3. Check reliability, verification state, red machine errors, and missing details such as ports, RAM, upload speed, and download speed. +4. Confirm the CLI is authenticated to the same host-enabled account. +5. Search directly for the machine ID; see [Why Isn't My Machine in Search?](/host/not-in-search#check-machine-listed). +6. If the console state looks wrong after refresh and rerun, collect logs and escalate to support. + + +## `nccl_failed` + +`nccl_failed` means the temporary self-test instance could not initialize and synchronize NCCL workers across the visible GPUs. Treat this as a multi-GPU communication failure, not as proof of one specific root cause. + +Start with: + +1. Inspect the self-test support bundle for the NCCL error output. +2. Confirm every GPU is visible with `nvidia-smi` and Docker GPU access. +3. Check driver/CUDA/PyTorch/NCCL compatibility. +4. Check GPU peer-to-peer communication, PCIe/NVLink topology, Xid errors, and any GPU reset or bus errors. +5. On HGX, DGX, or other NVSwitch systems, check Fabric Manager/NVLSM on the host. The self-test container can show NCCL symptoms, but it cannot confirm whether host services such as `nvidia-fabricmanager` are running or healthy. + +Useful host-side checks for NVSwitch systems include: + +```bash +systemctl status nvidia-fabricmanager +journalctl -u nvidia-fabricmanager --since "-24h" +nvidia-smi -q | grep -i -A 2 Fabric +nvidia-smi topo -m +``` + +Fabric state should normally show completed/successful registration on systems where NVIDIA exposes fabric state through NVML. If Fabric Manager is not applicable to the hardware, focus on driver/runtime compatibility, PCIe/NVLink topology, and the raw NCCL error. + + +## Which warnings can be ignored? + +Avoid ignoring warnings by default. One clear self-test advisory is that mapping more than the minimum number of direct ports per GPU is not itself a self-test gate. For production hosting, about 100 forwarded TCP/UDP ports per listed GPU is still practical headroom because individual instances can use many ports and old mappings may overlap with new starts. Red machine errors, self-test failures, NVML/Xid errors, port networking issues, and not-visible/rentable states should be investigated. + +## Reliability below the preflight gate + +Reliability greater than 0.9 is a self-test and verification gate; if the machine is at or below it, the self-test can fail before renting the temporary instance. See [Reliability & Uptime](/host/reliability-uptime#reliability-threshold). + +## Errors not specific to the self-test + +For `bad bandwidthtest2`, CDI device injection failures, NVML mismatches, and other machine-health errors, see [Machine Error Reference](/host/machine-errors) and [Common Errors & Diagnostics](/host/common-errors-diagnostics). diff --git a/host/storage-setup.mdx b/host/storage-setup.mdx new file mode 100644 index 00000000..524fd770 --- /dev/null +++ b/host/storage-setup.mdx @@ -0,0 +1,152 @@ +--- +title: "Storage Setup" +sidebarTitle: "Storage Setup" +description: "Disk, filesystem, Docker storage, XFS, RAID, and NVMe guidance for hosts." +"canonical": "/host/storage-setup" +personas: + - pro-operator + - headless-operator +--- + +
Pro OperatorHeadless / DC
+ +Vast uses Docker storage for renter data. Put `/var/lib/docker` on fast SSD/NVMe storage with XFS project quotas. + + +Formatting a disk or LVM volume destroys data. Identify the OS disk and any provider data mount before running `mkfs`. + + + +## Docker Storage Layout + +Current listing guidance expects at least 128 GB of fast SSD storage per GPU. Docker storage should: + +- Be on the intended data disk, not accidentally on the OS disk. +- Mount predictably at boot before Docker and the host daemon start. +- Use XFS with `pquota` or `prjquota`. +- Have enough capacity for active rentals, stopped instances, cached images, and any volume offers. + +Inspect first: + +```bash +lsblk -f +findmnt / +findmnt /data0 || true +findmnt /var/lib/docker || true +df -h / +``` + +Back up `/etc/fstab` before editing mounts: + +```bash +sudo cp /etc/fstab /etc/fstab.pre-vast-docker.$(date -u +%Y%m%d%H%M%S) +``` + + +## XFS Project Quotas + +XFS project quotas are required. Vast uses them for per-container storage quotas. + +Add `pquota` or `prjquota` to the mount-options column for the `/var/lib/docker` XFS mount in `/etc/fstab`. In the example below, the mount-options column is `rw,auto,pquota,nofail`: + +```text +UUID=a1b2c3d4-e5f6-7890-abcd-ef1234567890 /var/lib/docker xfs rw,auto,pquota,nofail 0 0 +``` + +After editing `/etc/fstab`, mount and verify: + +```bash +sudo mount -a +findmnt /var/lib/docker -no SOURCE,FSTYPE,OPTIONS +sudo xfs_quota -x -c "state" /var/lib/docker +``` + +The mount should be XFS, include `pquota` or `prjquota`, and show project quota accounting/enforcement as ON. + +Do not work around quota errors by removing Docker quota settings. Fix the filesystem and mount instead. + + +## Disposable Device Example + +Use this only after you have confirmed the target partition, RAID device, or LVM volume is safe to dedicate to Docker. + +```bash +export VAST_DOCKER_DEVICE=/dev/mapper/vg1-lv--1 +lsblk -f "$VAST_DOCKER_DEVICE" +findmnt -S "$VAST_DOCKER_DEVICE" || true +if findmnt -S "$VAST_DOCKER_DEVICE" >/dev/null; then + sudo umount "$VAST_DOCKER_DEVICE" +fi +sudo mkfs.xfs -f "$VAST_DOCKER_DEVICE" +sudo mkdir -p /var/lib/docker +sudo blkid "$VAST_DOCKER_DEVICE" +``` + +Comment out any old `/data0` line for the same device, add the `/var/lib/docker` fstab line, then mount: + +```bash +sudo mount -a +findmnt /var/lib/docker -no SOURCE,FSTYPE,OPTIONS +sudo xfs_quota -x -c "state" /var/lib/docker +``` + +If you are starting from a raw disk, create a partition first. The [Headless Install Path](/host/headless-install) includes a `cfdisk` example. + + +## Example fstab Line + +Use the disk UUID, not a device name like `/dev/nvme0n1p1`: + +```text +UUID=a1b2c3d4-e5f6-7890-abcd-ef1234567890 /var/lib/docker xfs rw,auto,pquota,nofail 0 0 +``` + +After editing: + +```bash +sudo mount -a +findmnt /var/lib/docker -no SOURCE,FSTYPE,OPTIONS +sudo xfs_quota -x -c "state" /var/lib/docker +``` + +The mount should be XFS, include `pquota` or `prjquota`, and show project quota accounting/enforcement as ON. + + +## Single NVMe vs RAID + +One NVMe is simplest. Multiple NVMes in RAID0 can improve capacity and throughput, but RAID0 has no redundancy: one failed drive can break the array. + +Redundant RAID layouts only help if you know when a device has failed. Set up disk health monitoring and failure notifications for the RAID layer, NVMe/SMART health, and any controller or storage software you use. Without notifications, a degraded array can leave you blind until another failure causes downtime or data loss. + +Pick the layout that matches your reliability goals, then monitor it like production storage. + + +## Docker pquota Error + +`--storage-opt is supported only for overlay over xfs with pquota` means Docker storage is not on XFS with project quotas. + +Move Docker's data root to a correctly mounted XFS filesystem, verify quotas, then restart Docker and the host software. + +## Installer Connection + +The Host Installer Wizard should detect prepared Docker storage. If you use the [headless fallback installer](/host/installing-host-software#headless-fallback), pass the block device behind the Docker mount: + +```bash +sudo python3 ./install "$VAST_HOST_KEY" \ + --no-driver \ + --docker-partition /dev/mapper/vg1-lv--1 \ + --ports 40000 40799 +``` + +Fix ext4, missing quotas, or wrong-device mounts before installing. + +## Volume Offers And Cached Images + +Rented volume space reduces storage available to GPU offers. Cached images can be reused when possible instead of redownloaded. + +For volume behavior, see [Hosting Overview](/host/hosting-overview#volume-offers). + + +## Does Vast redownload images that were already used? + +Prior images are cached when possible. If a renter uses an image already present on the machine, the system can reuse it. diff --git a/host/supported-hardware.mdx b/host/supported-hardware.mdx new file mode 100644 index 00000000..5019b53d --- /dev/null +++ b/host/supported-hardware.mdx @@ -0,0 +1,83 @@ +--- +title: "Supported Hardware" +sidebarTitle: "Supported Hardware" +description: "Hardware compatibility guidance before buying or listing a host machine." +"canonical": "/host/supported-hardware" +personas: + - hobbyist + - business-owner +--- + +
HobbyistBusiness
+ +Use this page before buying hardware, installing the host software, or listing a machine. + + +Meeting minimum requirements makes a machine eligible. It does not guarantee verification, search placement, rentals, or earnings. + + +## Quick Check + +A typical Vast host needs: + +- Native Ubuntu/Linux on x86_64/AMD64 hardware. +- Same-type GPUs. +- Supported NVIDIA GPUs, or the AMD families listed below. +- More than 7 GiB VRAM per GPU. +- AVX-capable CPU. +- At least one physical CPU core per visible/listed GPU. Hyperthreads do not count. +- Enough system RAM for the GPU tier. +- Fast SSD/NVMe storage, with at least 128 GB per GPU as a listing baseline. +- Public inbound networking with direct TCP and UDP port forwarding. + +## GPU Support + +| Vendor | Guidance | +| --- | --- | +| NVIDIA | 10-series or newer. Newer datacenter, workstation, and recent consumer GPUs are more attractive to renters. | +| AMD | MI25 or newer Radeon Instinct, Radeon VII, Radeon Pro VII, Radeon RX 7900 GRE/XT/XTX, and Radeon Pro W7900/W7800. Other 6000-series or newer Radeon RX/Pro W GPUs may work, but may not appear in standard ROCm filters. | + +Avoid mixed GPU models unless Vast explicitly documents support for that setup. + +If a new GPU model is not listed or searchable yet, ask Vast to add it through support or the [host community channels](/host/community). + +## Minimum Listing Baseline + +| Area | Baseline | +| --- | --- | +| OS | Ubuntu 24.04 LTS preferred; Ubuntu 22.04 also works. | +| Machine use | Dedicated host machine while rented. | +| CPU | AVX-capable, with at least one physical core per listed GPU. | +| System RAM | At least 4 GB per GPU as a listing baseline; high-VRAM GPUs may need more. | +| Storage | Fast SSD/NVMe, at least 128 GB per GPU. | +| PCIe | Enough lanes/bandwidth for the GPUs; avoid unstable risers, power, and BIOS layouts. | +| Network | Reliable public inbound connectivity for the configured port range. | + +For current verification thresholds, see [Verification Stages](/host/verification-stages#additional-requirements-for-verification). + +## Balance Matters + +A strong GPU can still be a poor host if the rest of the machine is weak. Match CPU, RAM, PCIe, storage, cooling, power, and network capacity to the GPU tier. + +Avoid reducing GPU count, RAM, disk, or other capacity after the machine is created; reductions can trigger deverification. + +## Storage And Network + +Docker storage should be fast, predictable, and mounted before Docker starts. Vast uses XFS project quotas for per-container storage limits. See [Storage Setup](/host/storage-setup). + +Hosts also need public inbound connectivity. CGNAT, double NAT without a real public forwarding path, blocked ports, IPv6-only service, LAN-only port forwarding, packet loss, or unstable upload can prevent listing, self-test, or rentals. See [Network & Ports](/host/network-ports). + +## Unsupported Or Discouraged Setups + +| Setup | Why | +| --- | --- | +| Windows or WSL as the host OS | Vast hosting is built for native Linux hosts. | +| ARM or GB10-class systems | Current host requirements assume AVX-capable x86_64/AMD64 hardware unless Vast documents an ARM path. | +| Mixed GPU models | Complicates offers, verification, and renter expectations. | +| Low-VRAM or very old GPUs | Self-test requires more than 7 GiB VRAM; demand may also be weak. | +| Gaming/workstation use during rentals | Rented machines should be dedicated. | +| No public inbound networking | Connectivity checks and renter access can fail. | + +## Before You Buy + +Confirm GPU demand with [GPU Market Metrics](/host/market-metrics), then verify CPU, RAM, PCIe, storage, and networking before spending money. After hardware is ready, continue with [Hosting Quickstart](/host/quickstart). diff --git a/host/understanding-verification.mdx b/host/understanding-verification.mdx index 044e025b..e3f40ab3 100644 --- a/host/understanding-verification.mdx +++ b/host/understanding-verification.mdx @@ -1,135 +1,77 @@ --- -title: Understanding Verification +title: "Understanding Verification" +sidebarTitle: "Understanding Verification" createdAt: Mon Nov 03 2025 17:30:00 GMT+0000 (Coordinated Universal Time) updatedAt: Mon Nov 03 2025 17:30:00 GMT+0000 (Coordinated Universal Time) "canonical": "/host/understanding-verification" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist --- +
All host personas
+ ## How Verification Works -Verification is **fully automated and powered by proprietary algorithms** that continuously evaluate each machine’s operational health and performance, dynamically adjusting assessments based on real-time **supply and demand**. - -Only machines that meet the platform's defined reliability and performance thresholds are eligible for verification. - -This process involves **no manual intervention**, ensuring consistent, scalable, and objective verification across all systems. - ---- - -To qualify, a machine must pass [minimum baseline](https://docs.vast.ai/host/how-to-self-test), and health/stability checks. Beyond that, the system evaluates four primary criteria (order not indicative of priority): +Verification is automated. A machine becomes eligible when it passes self-test and meets reliability, performance, hardware, network, and marketplace gates. -**Note:** Meeting the minimum requirements makes your machine eligible for verification, but it does not guarantee that verification will occur immediately. The final verification outcome still depends on the factors listed below. +Eligibility does not guarantee immediate verification, search placement, or rentals. -## 1) Reliability - -**Definition:** Stable, uninterrupted operation over time (uptime, resilience under continuous workloads). - -**Do** - -- Maintain consistent uptime with minimal downtime. -- Keep network connectivity stable; avoid jitter and drops. -- Manage thermals and power to prevent throttling. -- Proactively monitor hardware health and perform maintenance. - -**Avoid** - -- Frequent restarts or unplanned outages. -- Overheating, undervolting, or unstable power delivery. - -> **Note:** Higher reliability greatly improves verification likelihood. Sustained ≥99.99% (up to 99.9999%+) uptime is typically favored. - ---- - -## 2) Infrastructure Configuration +Verification is not permanent. Automation can move machines between Unverified, Verified, and Deverified as reliability, health, performance, network reachability, and marketplace conditions change. -**Definition:** Hardware, network, and software readiness to meet operational standards. +## Main Factors -### Hardware +| Factor | What matters | +| --- | --- | +| Reliability | The machine enters the verification pool when reliability is greater than 0.9. Higher reliability improves confidence. | +| Hardware balance | GPU model/count, VRAM, CPU, RAM, PCIe bandwidth, storage, and network should match the GPU tier. | +| Network | Stable bandwidth and reachable direct ports. | +| Software | Compatible NVIDIA/AMD driver stack, CUDA/runtime health, Docker/GPU runtime health, and a clean host. | +| DLPerf | Sustained real-world GPU throughput; PCIe, thermals, power, and drivers can affect it. | +| VM support | Optional, but can improve fit for some renter searches when enabled and supported. | +| Marketplace demand | In-demand GPUs, strong reliability, enough VRAM, and good network/location are more likely to be prioritized. | -- **GPU:** Type, memory, and count matter. Newer datacenter/workstation GPUs are prioritized (e.g., B200 > H200 >> 5090 > 4070). -- **VRAM:** More VRAM improves performance profiles. -- **GPU Count:** For the same GPU type, more GPUs increase verification likelihood (e.g., 8×5090 >> 2×5090 > 1×5090). -- **PCIe Bandwidth:** Adequate throughput is essential; bottlenecks depress DLPerf and overall performance. -- **CPU:** Favor strong, server-grade CPUs; actual measured performance matters. -- **Storage:** Both capacity and bandwidth (e.g., NVMe) impact responsiveness and reliability. + +## Supply And Demand -### Network +Verification also considers renter demand and available supply. A healthy machine can wait if demand for that configuration is low, while scarce or high-demand configurations may be prioritized. -- High-speed, symmetric, stable bandwidth is favored. -- Ensure required ports are open and accessible; a static IP helps. +To improve fit: -### Virtualization +- Use in-demand GPU models. +- Keep reliability high. +- Avoid under-provisioned CPU, RAM, PCIe, storage, or bandwidth. +- Enable VM support only when the machine supports it cleanly. +- Keep the system dedicated to Vast workloads. -- Enabling VM support significantly improves verification likelihood. + +## How Long Can Verification Take? -### Software +There is no fixed timeline. Passing self-test makes the machine eligible, but timing depends on automated checks, reliability, machine health, supply, demand, and platform policy. -- Drivers/CUDA must be correctly installed and compatible (use stable **latest** releases). -- Keep systems clean; run workloads via Create Job only. + +## Can Support Verify My Machine? -### System Optimization & Upgrades +No. Support can help with account issues or backend anomalies, but ordinary verification is not a manual support queue. Keep the machine healthy, eligible, and competitively listed while automation evaluates it. -- Balanced scaling matters (CPU/RAM/PCIe/bandwidth commensurate with GPU tier). -- Do not reduce hardware after creation (e.g., fewer GPUs/RAM) - this will trigger Deverified. -- Upgrades (adding GPUs/RAM) are allowed but may take time to reflect across the platform. +## Avoid -**Do** - -- Verify GPU PCIe connections provide full bandwidth and are not throttled. -- Keep the latest drivers/CUDA aligned with workloads. -- Confirm required ports and end-to-end reachability. - -**Avoid** - -- Pairing high-end GPUs with under-provisioned CPU/RAM. -- Letting hidden background services consume resources. - ---- - -## 3) DLPerf Score - -**Definition:** Estimated GPU performance on typical deep-learning tasks (e.g., CNN/Transformer training) for cross-hardware comparison. Higher DLPerf improves verification odds. [Read more](https://docs.vast.ai/guides/reference/faq/rental-types#dlperf-scoring) - -**Do** - -- Use the **latest** compatible drivers/CUDA. -- Eliminate PCIe, thermal, and power bottlenecks to maintain sustained clocks. - -**Avoid** - -- Misconfigurations that suppress benchmark performance. - ---- - -## 4) Supply & Demand Analysis - -**Definition:** Ongoing evaluation of market trends and renter behavior to surface configurations most likely to be rented. - -**Implication:** Machines aligned with active renter preferences-popular GPUs, sufficient VRAM, strong reliability, fast internet-are prioritized for verification to maximize utilization and profitability. - -**Do** - -- Offer in-demand GPU models with adequate VRAM and balanced resources. -- Maintain strong reliability to remain attractive once listed. - -**Avoid** - -- Niche/mismatched configurations with low renter interest. - ---- - -## Quick Reference - -| **Category** | **What Matters Most** | **How to Improve** | -|---------------|------------------------|--------------------| -| Reliability | Stable, uninterrupted uptime | Proactive monitoring; steady power/thermals; minimize restarts | -| Network | Symmetric, stable bandwidth; open ports | Upgrade links; verify routing/ports; monitor jitter/loss | -| Hardware | Modern GPUs/CPUs; adequate PCIe & RAM | Favor DC/workstation GPUs; ensure PCIe lanes; match CPU/RAM to GPU tier | -| Storage | Throughput and reliability | Prefer NVMe; monitor SMART; ensure sustained bandwidth | -| Virtualization | VM capability enabled | Enable in BIOS; enable IOMMU. | -| Software | Correct drivers/CUDA; clean system | Install latest, stable, compatible versions; use Create Job only | -| DLPerf | High real-world throughput | Fix PCIe/thermal bottlenecks; maintain clocks; correct drivers | -| Supply & Demand | Alignment with renter needs | Choose popular GPUs/VRAM; balance specs; maintain reliability | -| Upgrades | Changes reflected by platform | Scale up (add GPUs/RAM); avoid reductions that cause Deverified | +- Frequent restarts or unplanned outages. +- Closed ports, packet loss, or unstable upload. +- Thermal, power, PCIe, or driver instability. +- Background workloads outside Vast jobs/rentals. +- Reducing GPU count, RAM, disk, or other capacity after machine creation. + +## Read Next + +| Topic | Page | +| --- | --- | +| Exact requirements | [Verification Stages](/host/verification-stages#additional-requirements-for-verification) | +| Run diagnostics | [How to Self-Test](/host/how-to-self-test) | +| Reliability | [Reliability & Uptime](/host/reliability-uptime) | +| Market fit | [GPU Market Metrics](/host/market-metrics) | diff --git a/host/verification-stages.mdx b/host/verification-stages.mdx index 7976ce03..ce23ac83 100644 --- a/host/verification-stages.mdx +++ b/host/verification-stages.mdx @@ -1,137 +1,116 @@ --- -title: Verification Stages +title: "Verification Stages" +sidebarTitle: "Verification Stages" createdAt: Mon Jan 13 2025 21:20:40 GMT+0000 (Coordinated Universal Time) updatedAt: Mon Nov 03 2025 12:50:03 GMT+0000 (Coordinated Universal Time) "canonical": "/host/verification-stages" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist --- +
All host personas
-## States & Lifecycle +Verification is automated. There is no manual review step for ordinary host verification. -**States:** Unverified → Verified → (potentially) Deverified → Unverified → ... +## Lifecycle -**Lifecycle:** Machines automatically move between these states based on performance and reliability factors. Once verified, a machine will remain verified unless issues arise, such as failing health checks or reliability standards, which could lead to deverification. +Machines move between: ---- - -## How It Works - -Verification is **entirely automated by proprietary algorithms** that assess each machine’s operational health and performance, incorporating [supply-and-demand](https://docs.vast.ai/host/understanding-verification#4-supply-&-demand-analysis) dynamics. - -Only machines that meet the platform’s reliability and [performance thresholds](https://docs.vast.ai/host/how-to-self-test) are eligible for verification. There is **no manual intervention**, ensuring consistency, scalability, and objectivity. - ---- - -## Host Responsibilities (Always) +```text +Unverified -> Verified -> Deverified -> Unverified +``` -- Keep systems stable, well-cooled, and correctly configured. -- Maintain compatible drivers/CUDA and dependable, symmetric networking. -- Run jobs only through the Jobs tab or the `create job` CLI command. -- When issues arise, fix them promptly-the automation will update status. +Automation evaluates reliability, performance, hardware, network, and current marketplace demand. ---- +## Listing Minimums -## State Details & Guidance +To be listed, a machine should meet these baselines: -### Unverified - -**What it means:** Newly added machines or machines under evaluation. The system hasn't yet completed enough testing to confirm platform standards. This is not a judgment of quality-only that no platform guarantee exists yet. +```text +- Ubuntu 24.04 LTS preferred; Ubuntu 22.04 LTS also works +- Dedicated machine while rented +- Fast, reliable internet: at least 10 Mbps per machine +- Supported same-type GPUs +- At least 1 physical CPU core per GPU +- AVX-capable CPU +- At least 4 GB system RAM per GPU +- Fast SSD storage with at least 128 GB per GPU +- Sufficient PCIe bandwidth +- Open port range mapped to the machine +``` -**Do** -- [Pass the Self-Test](https://docs.vast.ai/host/how-to-self-test) -- Maintain steady uptime during evaluation. -- Ensure drivers/CUDA and networking are correctly installed and reachable. -- Keep the environment clean; schedule work via Create Job only. +For hardware detail, see [Supported Hardware](/host/supported-hardware). -**Avoid** -- Unnecessary reboots or configuration changes. -- Unrelated background workloads that consume GPU/CPU/IO. + +## Verification Requirements ---- +To enter the verification pool, the machine must also meet current self-test and platform gates: -### Minimum Guidelines for Listing on Vast.ai - -In order to be listed on Vast.ai, the machine must follow these minimum guidelines: - -```text Text -- Ubuntu 18.04 or newer (required) -- Dedicated machines only - the machine shouldn't be doing other stuff while rented -- Fast, reliable internet: at least 10Mbps per machine. -- 10-series Nvidia GPU or MI25 or newer Radeon Instinct series GPU or Radeon VII or Radeon Pro VII or Radeon RX 7900 (GRE/XT/XTX); or Radeon Pro W7900/W7800. Other 6000 series or newer Radeon RX/Pro W series GPUs may be supported; but may not be searchable using standard filters for AMD ROCm. -- At least 1 physical CPU core (2 hyperthreads) per GPU. -- Your CPU must support AVX instruction set (not all lower end CPUs support this). -- At least 4 GB of system RAM per GPU. -- Fast SSD storage with at least 128GB per GPU. -- At least 1X PCIe for every 2.5 TFLOPS of GPU performance. -- All GPUs on the machine must be of the same type. -- An open port range mapped to each machine. +```text +- CUDA version >= 11.8 +- Reliability > 0.9 +- At least 3 direct ports per GPU, forwarded for both TCP and UDP +- PCIe bandwidth > 2.85 GB/s +- Download and upload bandwidth scale with total VRAM +- GPU RAM > 7 GB per GPU +- System RAM at least 95% of total GPU VRAM, capped around 2 TB for very high-VRAM machines +- Passing self-test ``` ---- -### Additional Requirements for Verification +Bandwidth thresholds scale at about 2.6 Mbps per GiB of total VRAM, with a 100 Mbps floor and 500 Mbps ceiling. -In order for your unverified machine to be verified, it must also meet the following minimum requirements: +Reliability greater than 0.9 is the verification gate. Higher reliability improves confidence and can improve verification likelihood. See [Reliability & Uptime](/host/reliability-uptime#reliability-threshold). -```text Text -- CUDA version greater than or equal to 12.0 -- Reliability of 90% -- At least 3 open ports per GPU (100 recommended) -- Internet Download speed must scale with total machine VRAM at ~2.6 Mbps per GiB, with a 100 Mbps floor and 500 Mbps ceiling -- Internet Upload speed must scale with total machine VRAM at ~2.6 Mbps per GiB, with a 100 Mbps floor and 500 Mbps ceiling -- GPU RAM of 7 GB -- Passing the Self-Test -``` -> **Note:** High-end GPUs are more likely to be verified. Machines with datacenter GPUs such as B200, H200, H100, A100, etc., and those with premium GPUs such as RTX PRO 6000 WS, 8xRTX 5090, 8xRTX 4090, etc., receive prioritized verification processing due to their high demand and performance capabilities. - -**Tip:** Meeting these minimum requirements makes your machine eligible for verification, but does not automatically guarantee verification. [Read More about Verification](https://docs.vast.ai/host/understanding-verification) - ---- + +Meeting requirements makes the machine eligible. It does not guarantee immediate verification, search placement, or rentals. + +## Unverified -### Verified +Unverified means the machine is new, under evaluation, or not currently meeting verification conditions. -**What it means:** The machine passed automated checks for reliability, network stability, operational health, and performance. A Verified machine consistently delivers server services to platform standards. +Do: -**Do** -- Monitor health (uptime, thermals, power) and respond to alerts. -- Keep drivers/CUDA on compatible, **latest** stable versions. -- Maintain stable, symmetric bandwidth. +- Pass [self-test](/host/how-to-self-test). +- Keep uptime stable. +- Fix driver, network, storage, and daemon errors. +- Avoid background workloads outside the Jobs tab or `create job`. -**Avoid** -- Downgrading hardware capacity (e.g., reducing GPU count, disk or RAM). -- Allowing thermal, power, or bandwidth instability under load. +Avoid unnecessary reboots or hardware/config changes during evaluation. ---- +## Verified + +Verified means the machine passed automated checks and currently meets platform standards. -### Deverified +Keep it verified by: -**What it means:** -A previously Verified machine no longer meets requirements. System continuous monitoring detects sustained degradation. +- Monitoring thermals, power, drivers, Docker, and daemon health. +- Keeping networking stable and reachable. +- Avoiding hardware reductions after the machine is created. +- Responding quickly to red machine errors. -**When will deverification happen?** -- When the hosting software detects an error, your machine is automatically, but **temporarily**, deverified. It will appear as *Unverified* in search results until the underlying issue is resolved. +## Deverified -**How should I begin fixing it?** -- A **red error indicator** will appear on your machine in the Machines tab. Use this message to identify and investigate the issue in your logs or metrics. +Deverified means a previously verified machine no longer meets requirements. It appears as Unverified until the issue clears. -**Recovery:** -Fix the issue and restore stability; the system will **automatically** transition back to Verified once system confirms healthy operation. -This process may take some time, as the system ensures that the issue is fully resolved before restoring verification. Most error messages are cleared within 1-2 hours of resolution. +Common causes: -**Common causes** - Network instability, closed ports, or low bandwidth. -- Hardware/system errors (e.g., failing storage, insufficient PCIe bandwidth). -- GPU issues (e.g., nvidia-smi/NVML failures, container device init errors). -- Container launch failures or repeated runtime exceptions. -- Detected abuse or policy violations. +- GPU, NVML, Xid, PCIe, power, thermal, or storage errors. +- Container launch failures. +- Reliability dropping to or below the gate. +- Policy or abuse-related flags. -**Do** -- Investigate red error indicators quickly; review logs and metrics. -- Validate thermal/power headroom and bandwidth under load. -- Re-check health after changes to confirm resolution. +Fix the underlying issue and wait for automation to refresh state. Most machine error states clear only after the platform sees sustained healthy behavior. -**Avoid** -- Ignoring warnings or allowing instability to persist. -- Reducing hardware below the created specification. +## Read Next -> **Note:** If you’ve fixed the issue but the system doesn’t automatically detect the resolution, a Vast.ai team member may need to manually check that your machine is functioning correctly and clear the error. +| Topic | Page | +| --- | --- | +| What verification means | [Understanding Verification](/host/understanding-verification) | +| Running self-test | [How to Self-Test](/host/how-to-self-test) | +| Generated self-test checks | [Self-Test Reference](/host/self-test-reference) | +| Direct ports | [Network & Ports](/host/network-ports) | diff --git a/host/vms.mdx b/host/vms.mdx index 82abdf2f..1b2cb3ba 100644 --- a/host/vms.mdx +++ b/host/vms.mdx @@ -1,123 +1,100 @@ --- -title: VMs +title: "VMs & IOMMU" +sidebarTitle: "VMs & IOMMU" createdAt: Mon Jan 13 2025 21:20:40 GMT+0000 (Coordinated Universal Time) updatedAt: Wed Jan 15 2025 00:37:08 GMT+0000 (Coordinated Universal Time) +personas: + - pro-operator + - headless-operator --- +
Pro OperatorHeadless / DC
+ +Vast can run KVM virtual machines in addition to Docker containers. VM support is optional and depends heavily on hardware, BIOS, kernel, and GPU configuration. -**WARNING:** -VMs interface much more directly with hardware than Docker containers. -Proper VM support is very sensitive to hardware setup. -This guide covers the configuration steps needed to enable support for Vast VMs on most setups, but is not and cannot be exhausitve. +VMs interact more directly with hardware than containers. Enable them only when the machine supports IOMMU and remains stable. -# Introduction - -Vast now supports VM instances running on Kernel Virtual Machine (KVM) in addition to Docker container based instances. -VM support is currently an optional feature for hosts as it usually requires additional configuration steps on top of those needed to support Docker-based instances. - -Host machines are not required to be VM compatible; the Vast hosting software will automatically test and enable the feature on machines on which VMs are supported. -On new machines the tests will be run on install; for machines configured before the VM-feature release, testing for VM-compatability will happen when the machine is unoccupied. - -Machines that do not have VM support enabled will be hidden in the search page for clients who have VM-based templates selected. - -## VM Support Benefits/Drawbacks - -### Benefits - -VM support will allow your machine to take advantage of demand for use cases that Docker cannot easily support, in addition to demand for conventional Docker-based instances. - -VMs support the following features/use-cases that Docker-based instances do not: - - -

Feature

-

Use-case

- - -Systemd/Docker - -Multi-Application Server Tooling and DevOps (e.g., Docker Compose, Kubernetes, Docker Build) - -Non-Linux OSes - -Windows Graphics (e.g., for rendering or cloud gaming) - -ptrace - -Program analysis for CUDA-performance optimization (e.g., via Nvidia NSight) - -Currently no other peer-to-peer GPU rental marketplace offers full VMs; instead full VMs are only available from traditional providers at much higher costs. -Thus we believe that hosts who have VMs enabled can expect to command a substantial preumium. - -### Drawbacks - -- Due to greater user control over hardware, VM support requires IOMMU settings for securing PCIe communications that can degrade the performance of NCCL on non-RTX 40X0 multi-GPU machines that rely on PCI-based GPU peer-to-peer communication. -- VMs require more disk space than Docker containers as they do not share components with the host OS. Hosts with VMs enabled may want to set higher disk and internet bandwidth prices. - -### Summary +Default host onboarding should still start from the official [Vast host setup page](https://cloud.vast.ai/host/setup/). Use this page only for optional VM/IOMMU planning or troubleshooting after the normal host setup path is understood. -We recommend all hosts with single-GPU rigs to try to ensure VM support as the drawbacks for single-GPU machines are minimal. +## When VM Support Helps -We also generally recommend multi-GPU Hosts with RTX 40X0 series GPUs try enabling VMs, especially if they have plentiful disk space and fast (500Mbps+) internet speed, -as rendering/gaming users will benefit from those, as well as users who need multi-application orchestration tools. +VM support can make a host visible to renters using VM templates and can help with workloads that need: -We do not recommend multi-GPU hosts with datacenter GPUs enable VMs until we can ensure better GPU P2P communication support in VMs, including support for NVLink. +- Systemd, Docker Compose, Kubernetes, or Docker build workflows. +- Non-Linux OS images. +- `ptrace` tooling such as NVIDIA Nsight. -## Configuring VMs on your machine +Single-GPU hosts usually have the lowest VM risk. Multi-GPU RTX 40-series hosts may also benefit. Multi-GPU datacenter hosts should be more cautious until VM P2P/NVLink behavior is a good fit for the workload. -### Checking VM enablement status. +## Tradeoffs -Run `python3 /var/lib/vastai_kaalia/enable_vms.py check`. +- IOMMU settings can reduce PCIe peer-to-peer performance on some multi-GPU systems. +- VMs can use more disk than containers. +- Hosts with VM support may need higher disk and bandwidth prices. -Possible results are: +## Check VM Status -- `on`: VMs are enabled on your machine. -- `off`: VMs are disabled on your machine. Either you disabled VMs or our previous tests failed. -- `pending`: VMs are not disabled, but will try to enable once the machine is idle. +```bash +python3 /var/lib/vastai_kaalia/enable_vms.py check +``` -### Disabling VMs. +Results: -To prevent VMs from being enabled on your machine, or to disable VMs after they have been enabled, run `python3 /var/lib/vastai_kaalia/enable_vms.py off`. +| Status | Meaning | +| --- | --- | +| `on` | VM support is enabled. | +| `off` | VM support is disabled or a previous test failed. | +| `pending` | VM support will be tested when the machine is idle. | -Note that default configuration settings for most machines will not support VMs, and we can detect that, so most hosts who do not want VMs enabled do not need to take any action. +## Disable VM Support -### Configuring your machine to support VMs. +```bash +python3 /var/lib/vastai_kaalia/enable_vms.py off +``` -### Hardware prerequisites +Most hosts do not need to disable anything; unsupported machines are detected automatically. -You will require a CPU and a chipset that support Intel VT-d or AMD-Vi. +## Hardware Requirements -### Configure BIOS +VM support requires CPU/chipset support for Intel VT-d or AMD-Vi, plus BIOS virtualization support. -Check that virtualization is enabled in your BIOS. On most machines, this should be enabled by default. +In BIOS, enable virtualization and IOMMU/VT-d/AMD-Vi options. -### Configure Kernel Commandline Arguments +## Kernel Options -For further reference refer to [Preparing the IOMMU](https://ubuntu.com/server/docs/gpu-virtualization-with-qemu-kvm#preparing-the-input-output-memory-management-unit-iommu). +Edit `/etc/default/grub` and add the appropriate IOMMU option plus NVIDIA modeset disablement: -We will need to ensure IOMMU, a technology that secures and isolates communication between PCIe devices, is set up, along with disabling all driver features that interfere with VMs. +```text +GRUB_CMDLINE_LINUX="intel_iommu=on nvidia_drm.modeset=0" +``` -Open `/etc/default/grub` and add to the `GRUB_CMDLINE_LINUX=` the following: +Use `amd_iommu=on` instead of `intel_iommu=on` on AMD platforms. -- `amd_iommu=on` or `intel_iommu=on` depending on whether you have an AMD or Intel CPU. -- `nvidia_drm.modeset=0` +Some hosts may also need: -Some hosts may also need to add the following settings: +```text +rd.driver.blacklist=nouveau modprobe.blacklist=nouveau +``` -- `rd.driver.blacklist=nouveau` -- `modprobe.blacklist=nouveau` +Apply and reboot: -Then run `sudo update-grub` and reboot. +```bash +sudo update-grub +sudo reboot +``` -### Disable display managers/background GPU processes. +## Display Managers And GPU Processes -If you have a display manager (e.g., GDM) or display server (XOrg, Wayland, etc) running, you must disable them. +Disable display managers, XOrg/Wayland sessions, and background GPU processes. `nvidia-persistenced` is okay because it is managed by the host stack. -You may not run any background GPU processes for VMs to work (`nvidia-persitenced` is OK, it is managed by our hosting software). +## Retry Enablement -### Enabling VMs +If VM support is off and you want to retry after fixing the host: -We will check/test your configuration when your machine is idle and enable VMs by default if your machine is capable of supporting VMs, and you have not set VMs to `off`. +```bash +sudo python3 /var/lib/vastai_kaalia/enable_vms.py on -f +``` -If you have VMs set to off, and you'd like to retry enabling VMs, run `sudo python3 /var/lib/vastai_kaalia/enable_vms.py on -f` while your machine is idle. +Run this only when the machine is idle. diff --git a/host/workload-policy.mdx b/host/workload-policy.mdx new file mode 100644 index 00000000..c5837ae9 --- /dev/null +++ b/host/workload-policy.mdx @@ -0,0 +1,76 @@ +--- +title: "Workload Policy" +sidebarTitle: "Workload Policy" +description: "Host-facing expectations for workload types and platform policy." +"canonical": "/host/workload-policy" +personas: + - pro-operator + - headless-operator + - business-owner + - hobbyist +--- + +
All host personas
+ +This is a host-operator summary. The source of truth is always the [Host Agreement](https://cloud.vast.ai/host/agreement) and [Terms of Service](https://vast.ai/terms). + + +Do not use this page as legal advice or as a replacement for the agreement and terms. + + +## Host Responsibilities + +While a rental contract is active: + +- Keep the advertised machine online, stable, and performant. +- Do not run local gaming, mining, benchmarks, display workloads, or background GPU jobs. +- Do not stop, inspect, modify, or interfere with renter containers because they appear idle. +- Keep renter data isolated. +- Keep escalation contact details current. +- Follow the host agreement, terms, export rules, and applicable law. + +An idle-looking rental can still be active. Renters may run notebooks, services, build jobs, CPU-heavy work, or bursty workloads. + + +## Can I Message A Client? + +No. Hosts do not have an established direct client-message flow. If a workload looks broken, idle, abusive, or policy-sensitive, follow the escalation guidance below. + +## Restricted Activity + +The Terms describe prohibited activity. Host-facing examples include: + +| Category | Host action | +| --- | --- | +| Illegal or abusive activity | Preserve observable details and escalate. | +| Malware, spam, network abuse, or attacks | Do not knowingly support it; escalate. | +| Platform abuse or security bypass | Escalate suspected attempts to bypass controls or disrupt systems. | +| Intellectual-property concerns | Escalate credible concerns; do not inspect private renter data. | +| Export-controlled or sanctioned use | Escalate suspected restricted users, destinations, or end uses. | +| Cryptocurrency mining | Do not advertise mining as generally allowed. Check the Terms or ask support. | + +## Mining And Crypto-Adjacent Workloads + +Some blockchain-adjacent workloads are AI inference or training; others may be mining, proof-of-work, or policy-sensitive token activity. + +Hosts should not decide policy on behalf of Vast. If a renter asks whether a specific crypto workload is allowed, point them to the [Terms of Service](https://vast.ai/terms) or support. + +## What To Do + +| Situation | Action | +| --- | --- | +| Rental appears idle | Leave it running. | +| Rental affects machine stability | Check host hardware, thermals, power, drivers, Docker, and daemon logs. | +| Suspected abuse | Collect timestamps, symptoms, screenshots, and visible instance details; contact support. | +| Planned maintenance needed | Use [maintenance windows](/host/maintenance-windows) and offer-end-date planning. | +| Stop new rentals | Unlist the machine; existing contracts continue. | +| Expired contract or stale storage | Use documented cleanup flows in [Remove or Recreate](/host/removing-recreating-machines). | + +## Related Pages + +| Topic | Read next | +| --- | --- | +| Contract behavior | [Hosting Overview](/host/hosting-overview) | +| Host agreement | [Host Agreement](https://cloud.vast.ai/host/agreement) | +| Terms | [Terms of Service](https://vast.ai/terms) | +| Maintenance | [Maintenance Windows](/host/maintenance-windows) | diff --git a/images/host-installer-wizard-tui.png b/images/host-installer-wizard-tui.png new file mode 100644 index 00000000..0a7a45fd Binary files /dev/null and b/images/host-installer-wizard-tui.png differ diff --git a/persona-chips.css b/persona-chips.css new file mode 100644 index 00000000..21ad54a8 --- /dev/null +++ b/persona-chips.css @@ -0,0 +1,23 @@ +/* Persona tag chips — rendered top-right of host doc pages. + Mintlify applies any .css file in the repo automatically. */ + +.persona-chips { + display: flex; + justify-content: flex-end; + flex-wrap: wrap; + gap: 0.4rem; + margin: 0 0 1.25rem 0; +} + +.persona-chip { + font-size: 0.72rem; + font-weight: 500; + line-height: 1; + letter-spacing: 0.02em; + padding: 0.32rem 0.7rem; + border-radius: 9999px; + border: 1px solid rgba(128, 128, 128, 0.35); + background: rgba(128, 128, 128, 0.08); + opacity: 0.9; + white-space: nowrap; +} diff --git a/snippets/host/cli/defrag-machines.mdx b/snippets/host/cli/defrag-machines.mdx index 285eb549..5cfb3789 100644 --- a/snippets/host/cli/defrag-machines.mdx +++ b/snippets/host/cli/defrag-machines.mdx @@ -5,7 +5,7 @@ Defragment machines ## Usage ```bash -vastai defragment machines IDs +vastai defrag machines IDs ``` ## Arguments diff --git a/snippets/host/cli/list-machines.mdx b/snippets/host/cli/list-machines.mdx index 92c32fe4..faddac39 100644 --- a/snippets/host/cli/list-machines.mdx +++ b/snippets/host/cli/list-machines.mdx @@ -11,7 +11,7 @@ vastai list machines IDs [options] ## Arguments - ids of instance to list + ids of machines to list ## Options diff --git a/snippets/host/cli/schedule-maint.mdx b/snippets/host/cli/schedule-maint.mdx index 196ca1ba..7d7e4af5 100644 --- a/snippets/host/cli/schedule-maint.mdx +++ b/snippets/host/cli/schedule-maint.mdx @@ -5,7 +5,7 @@ Schedule upcoming maint window ## Usage ```bash -vastai schedule maintenance id [--sdate START_DATE --duration DURATION --maintenance_category MAINTENANCE_CATEGORY] +vastai schedule maint id [--sdate START_DATE --duration DURATION --maintenance_category MAINTENANCE_CATEGORY] ``` ## Arguments diff --git a/snippets/host/cli/self-test-machine.mdx b/snippets/host/cli/self-test-machine.mdx index 94727b4a..98259301 100644 --- a/snippets/host/cli/self-test-machine.mdx +++ b/snippets/host/cli/self-test-machine.mdx @@ -5,7 +5,7 @@ Perform a self-test on the specified machine ## Usage ```bash -vastai self-test machine [--debugging] [--explain] [--api_key API_KEY] [--url URL] [--retry RETRY] [--raw] [--ignore-requirements] +vastai self-test machine [--debugging] [--ignore-requirements] [--test-image IMAGE] [--support-bundle-dir DIR] [--no-support-bundle] ``` ## Arguments @@ -24,6 +24,18 @@ vastai self-test machine [--debugging] [--explain] [--api_key API_K Ignore the minimum system requirements and run the self test regardless + + Use a custom self-test image for testing custom self-test images. Overrides environment and CUDA image mapping. + + + + Directory for failure diagnostic bundles. Defaults to `/tmp`. + + + + Do not create a diagnostic tarball when the self-test fails. + + ## Description This command tests if a machine meets specific requirements and @@ -33,9 +45,8 @@ runs a series of tests to ensure it's functioning correctly. ```bash vast self-test machine 12345 - vast self-test machine 12345 --debugging - vast self-test machine 12345 --explain - vast self-test machine 12345 --api_key +vast self-test machine 12345 --debugging +vast self-test machine 12345 --support-bundle-dir /tmp/vast-bundles ``` ## Global Options diff --git a/snippets/host/cli/set-min-bid.mdx b/snippets/host/cli/set-min-bid.mdx index 96817f56..8733c483 100644 --- a/snippets/host/cli/set-min-bid.mdx +++ b/snippets/host/cli/set-min-bid.mdx @@ -5,7 +5,7 @@ Set the minimum bid/rental price for a machine ## Usage ```bash -vastai set min_bid id [--price PRICE] +vastai set min-bid id [--price PRICE] ``` ## Arguments diff --git a/snippets/host/cli/show-maints.mdx b/snippets/host/cli/show-maints.mdx index ecc1d31e..bc6521d0 100644 --- a/snippets/host/cli/show-maints.mdx +++ b/snippets/host/cli/show-maints.mdx @@ -5,14 +5,14 @@ Show maintenance information for host machines ## Usage ```bash -vastai show maints -ids 'machine_id_1' [OPTIONS] -vastai show maints -ids 'machine_id_1, machine_id_2' [OPTIONS] +vastai show maints --ids 'machine_id_1' [OPTIONS] +vastai show maints --ids 'machine_id_1,machine_id_2' [OPTIONS] ``` ## Options - - comma seperated string of machine_ids for which to get maintenance information + + comma separated string of machine IDs for which to get maintenance information (alias: `--ids`) @@ -22,7 +22,7 @@ vastai show maints -ids 'machine_id_1, machine_id_2' [OPTIONS] ## Examples ```bash -vastai show maints +vastai show maints --ids ``` ## Global Options diff --git a/style.css b/style.css index 9cd7ea44..f1d57767 100644 --- a/style.css +++ b/style.css @@ -3,6 +3,17 @@ * Mintlify automatically loads a root-level style.css on every page. */ +/* + * Keep hash-linked sections below the sticky Mintlify header. + * + * Many host pages use explicit `` markers before headings so + * support replies and table-of-contents links can target stable anchors. The + * sticky top navigation otherwise covers the first lines after a hash jump. + */ +:where(h1, h2, h3, h4, h5, h6, a[id]) { + scroll-margin-top: 8rem; +} + /* * Use the full viewport width on large screens. *